Make Machine a first class citizen (#794)

equinor · Dec 17, 2019 · 994a9f8 · 994a9f8
1 parent 3850be7
commit 994a9f8
Show file tree

Hide file tree

Showing 128 changed files with 927 additions and 1,040 deletions.
diff --git a/docs/components/dataset.rst b/docs/components/dataset.rst
diff --git a/docs/components/data_provider.rst → docs/components/machine/data_provider.rst b/docs/components/data_provider.rst → docs/components/machine/data_provider.rst
@@ -3,35 +3,35 @@ Data Providers & Readers
 
 Data Providers
 ==============
-.. automodule:: gordo_components.data_provider.providers
+.. automodule:: gordo_components.machine.dataset.data_provider.providers
     :members:
     :undoc-members:
     :show-inheritance:
 
 IROC Reader
 ===========
-.. automodule:: gordo_components.data_provider.iroc_reader
+.. automodule:: gordo_components.machine.dataset.data_provider.iroc_reader
     :members:
     :undoc-members:
     :show-inheritance:
 
 NCS Reader
 ==========
-.. automodule:: gordo_components.data_provider.ncs_reader
+.. automodule:: gordo_components.machine.dataset.data_provider.ncs_reader
     :members:
     :undoc-members:
     :show-inheritance:
 
 Azure Utils
 ===========
-.. automodule:: gordo_components.data_provider.azure_utils
+.. automodule:: gordo_components.machine.dataset.data_provider.azure_utils
     :members:
     :undoc-members:
     :show-inheritance:
 
 Base Data Provider
 ==================
-.. automodule:: gordo_components.data_provider.base
+.. automodule:: gordo_components.machine.dataset.data_provider.base
     :members:
     :undoc-members:
     :show-inheritance:
diff --git a/docs/components/machine/dataset.rst b/docs/components/machine/dataset.rst
@@ -0,0 +1,31 @@
+Datasets
+--------
+
+Datasets
+========
+.. automodule:: gordo_components.machine.dataset.datasets
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Base Dataset
+============
+.. automodule:: gordo_components.machine.dataset.base
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+DataProvider
+============
+
+Each dataset requires a data provider, which is
+responsible to provide the data to the logic of
+the dataset.
+
+.. toctree::
+    :maxdepth: 4
+    :caption: DataProvider
+
+    ./data_provider.rst
+
diff --git a/docs/components/machine/machine.rst b/docs/components/machine/machine.rst
@@ -0,0 +1,31 @@
+Machine
+-------
+
+A Machine is the central unity of
+a model, dataset, metadata and everything
+needed to create and build a ML model to
+be served by a deployment.
+
+.. automodule:: gordo_components.machine.machine.Machine
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Model
+=====
+
+.. toctree::
+    :maxdepth: 4
+    :caption: Model
+
+    ./model/model.rst
+
+Dataset
+=======
+
+.. toctree::
+    :maxdepth: 4
+    :caption: Dataset
+
+    ./dataset.rst
+
diff --git a/docs/components/model/anomaly.rst → docs/components/machine/model/anomaly.rst b/docs/components/model/anomaly.rst → docs/components/machine/model/anomaly.rst
@@ -10,7 +10,7 @@ AnomalyDetectorBase
 
 The base class for all other anomaly detector models
 
-.. automodule:: gordo_components.model.anomaly.base
+.. automodule:: gordo_components.machine.model.anomaly.base
     :members:
     :undoc-members:
     :show-inheritance:
@@ -22,7 +22,7 @@ DiffBasedAnomalyDetector
 Calculates the absolute value prediction differences between y and yhat as well
 as the absolute difference error between both matrices via ``numpy.linalg.norm(..., axis=1)``
 
-.. automodule:: gordo_components.model.anomaly.diff
+.. automodule:: gordo_components.machine.model.anomaly.diff
     :members:
     :undoc-members:
     :show-inheritance:
diff --git a/docs/components/model/model-factories.rst → ...ponents/machine/model/model-factories.rst b/docs/components/model/model-factories.rst → ...ponents/machine/model/model-factories.rst
@@ -14,15 +14,15 @@ that function will be used to create the underlying Keras model for
 feedforward factories
 =====================
 
-.. automodule:: gordo_components.model.factories.feedforward_autoencoder
+.. automodule:: gordo_components.machine.model.factories.feedforward_autoencoder
     :members:
     :undoc-members:
     :show-inheritance:
 
 lstm factories
 ==============
 
-.. automodule:: gordo_components.model.factories.lstm_autoencoder
+.. automodule:: gordo_components.machine.model.factories.lstm_autoencoder
     :members:
     :undoc-members:
     :show-inheritance:
diff --git a/docs/components/model/model.rst → docs/components/machine/model/model.rst b/docs/components/model/model.rst → docs/components/machine/model/model.rst
@@ -16,7 +16,7 @@ The base model is designed to be inherited from any other models which need
 to be implemented within Gordo due to special model requirements. ie. PyTorch,
 Keras, etc.
 
-.. automodule:: gordo_components.model.base
+.. automodule:: gordo_components.machine.model.base
     :members:
     :undoc-members:
     :show-inheritance:
@@ -26,10 +26,10 @@ Custom Gordo models
 
 This group of models are already implemented and ready to be used within
 config files, by simply specifying their full path. For example:
-``gordo_components.model.models.KerasAutoEncoder``
+``gordo_components.machine.model.models.KerasAutoEncoder``
 
 
-.. automodule:: gordo_components.model.models
+.. automodule:: gordo_components.machine.model.models
     :members:
     :undoc-members:
     :show-inheritance:
@@ -50,7 +50,7 @@ Utils
 Shared utility functions used by models and other components interacting
 with the model's.
 
-.. automodule:: gordo_components.model.utils
+.. automodule:: gordo_components.machine.model.utils
     :members:
     :undoc-members:
     :show-inheritance:
diff --git a/docs/components/model/transformer-funcs.rst → ...nents/machine/model/transformer-funcs.rst b/docs/components/model/transformer-funcs.rst → ...nents/machine/model/transformer-funcs.rst
@@ -7,7 +7,7 @@ A collection of functions which can be referenced within the
 General
 =======
 
-.. automodule:: gordo_components.model.transformer_funcs.general
+.. automodule:: gordo_components.machine.model.transformer_funcs.general
     :members:
     :undoc-members:
     :show-inheritance:
diff --git a/docs/components/model/transformers.rst → ...components/machine/model/transformers.rst b/docs/components/model/transformers.rst → ...components/machine/model/transformers.rst
@@ -8,7 +8,7 @@ inserted into ``Pipeline`` objects.
 Imputers
 ========
 
-.. automodule:: gordo_components.model.transformers.imputer
+.. automodule:: gordo_components.machine.model.transformers.imputer
     :members:
     :undoc-members:
     :show-inheritance:
diff --git a/docs/index.rst b/docs/index.rst
@@ -25,10 +25,8 @@ and deployed as a REST API on Kubernetes.
     :maxdepth: 4
     :caption: Components:
 
-    ./components/model/model.rst
+    ./components/machine/machine.rst
     ./components/builder.rst
-    ./components/data_provider.rst
-    ./components/dataset.rst
     ./components/serializer.rst
     ./components/server/server.rst
     ./components/cli.rst

diff --git a/examples/Gordo-Workflow-High-Level.ipynb b/examples/Gordo-Workflow-High-Level.ipynb
@@ -29,7 +29,7 @@
     "\n",
     "from gordo_components import serializer\n",
     "from gordo_components.builder import local_build\n",
-    "from gordo_components.data_provider.providers import DataLakeProvider"
+    "from gordo_components.machine.dataset.data_provider.providers import DataLakeProvider"
    ]
   },
   {
@@ -63,7 +63,7 @@
     "    metadata:\n",
     "      information: Some sweet information about the model\n",
     "    model:\n",
-    "      gordo_components.model.anomaly.diff.DiffBasedAnomalyDetector:\n",
+    "      gordo_components.machine.model.anomaly.diff.DiffBasedAnomalyDetector:\n",
     "        base_estimator:\n",
     "          sklearn.pipeline.Pipeline:\n",
     "            steps:\n",
@@ -166,17 +166,17 @@
       "  resolution: 10T\n",
       "  row_filter_buffer_size: 0\n",
       "  tag_list:\n",
-      "  - !!python/object/new:gordo_components.dataset.sensor_tag.SensorTag\n",
+      "  - !!python/object/new:gordo_components.machine.dataset.sensor_tag.SensorTag\n",
       "    - SOME-TAG1\n",
       "    - asgb\n",
-      "  - !!python/object/new:gordo_components.dataset.sensor_tag.SensorTag\n",
+      "  - !!python/object/new:gordo_components.machine.dataset.sensor_tag.SensorTag\n",
       "    - SOME-TAG2\n",
       "    - asgb\n",
       "  target_tag_list:\n",
-      "  - !!python/object/new:gordo_components.dataset.sensor_tag.SensorTag\n",
+      "  - !!python/object/new:gordo_components.machine.dataset.sensor_tag.SensorTag\n",
       "    - SOME-TAG3\n",
       "    - asgb\n",
-      "  - !!python/object/new:gordo_components.dataset.sensor_tag.SensorTag\n",
+      "  - !!python/object/new:gordo_components.machine.dataset.sensor_tag.SensorTag\n",
       "    - SOME-TAG4\n",
       "    - asgb\n",
       "  train_end_date: 2019-03-01 00:00:00+00:00\n",
@@ -449,7 +449,7 @@
       "  data-query-duration-sec: 0.0615231990814209\n",
       "  model-builder-version: 0.37.0rc2.dev8+ga427396.d20191021\n",
       "  model-config:\n",
-      "    gordo_components.model.anomaly.diff.DiffBasedAnomalyDetector:\n",
+      "    gordo_components.machine.model.anomaly.diff.DiffBasedAnomalyDetector:\n",
       "      base_estimator:\n",
       "        sklearn.pipeline.Pipeline:\n",
       "          steps:\n",

diff --git a/examples/Gordo-Workflow-Semi-Low-Level.ipynb b/examples/Gordo-Workflow-Semi-Low-Level.ipynb
@@ -25,8 +25,8 @@
     "\n",
     "from datetime import datetime\n",
     "\n",
-    "from gordo_components.dataset.datasets import TimeSeriesDataset\n",
-    "from gordo_components.data_provider.providers import DataLakeProvider\n",
+    "from gordo_components.machine.dataset.datasets import TimeSeriesDataset\n",
+    "from gordo_components.machine.dataset.data_provider.providers import DataLakeProvider\n",
     "from gordo_components import serializer\n",
     "\n",
     "data_provider = DataLakeProvider(storename=\"dataplatformdlsprod\", interactive=True)\n",
@@ -156,7 +156,7 @@
      "data": {
       "text/plain": [
        "Pipeline(memory=None,\n",
-       "     steps=[('step_0', MinMaxScaler(copy=True, feature_range=(0, 1))), ('step_1', <gordo_components.model.models.KerasAutoEncoder object at 0x7f64bd6de7f0>)])"
+       "     steps=[('step_0', MinMaxScaler(copy=True, feature_range=(0, 1))), ('step_1', <gordo_components.machine.model.models.KerasAutoEncoder object at 0x7f64bd6de7f0>)])"
       ]
      },
      "execution_count": 4,
@@ -170,7 +170,7 @@
     "    sklearn.pipeline.Pipeline:\n",
     "        steps:\n",
     "          - sklearn.preprocessing.data.MinMaxScaler\n",
-    "          - gordo_components.model.models.KerasAutoEncoder:\n",
+    "          - gordo_components.machine.model.models.KerasAutoEncoder:\n",
     "              kind: feedforward_hourglass\n",
     "    \"\"\"\n",
     ")\n",

diff --git a/examples/config_crd.yaml b/examples/config_crd.yaml
@@ -13,19 +13,19 @@ spec:
       - name: ct-23-0001 #1st machine
         dataset:
           tags: #list of tags for 1st machine
-            - TAG 1
-            - TAG 2
-            - TAG 3
+            - GRA-TAG 1
+            - GRA-TAG 2
+            - GRA-TAG 3
           train_start_date: 2016-11-07T09:11:30+01:00
           train_end_date: 2018-09-15T03:01:00+01:00
 
       - name: ct-23-0002 #2nd machine
         dataset:
           resolution: 2T
           tags: #list of tags for 2nd machine
-            - TAG 1
-            - TAG 2
-            - TAG 3
+            - GRA-TAG 1
+            - GRA-TAG 2
+            - GRA-TAG 3
           train_start_date: 2018-05-20T01:00:04+02:00
           train_end_date: 2019-05-10T15:05:50+02:00
         runtime:
@@ -39,9 +39,9 @@ spec:
       - name: ct-23-0003 #3rd machine
         dataset:
           tags: #list of tags for 3rd machine
-            - TAG 1
-            - TAG 2
-            - TAG 3
+            - GRA-TAG 1
+            - GRA-TAG 2
+            - GRA-TAG 3
           train_start_date: 2018-09-15T13:03:04+02:00
           train_end_date: 2019-03-11T11:05:10+02:00
 
@@ -66,10 +66,10 @@ spec:
               memory: 2000 #Kill the server if it uses more than 2GB
               cpu: 2000 #Throttle the server to never using more than 2 cores
       model:
-        gordo_components.model.anomaly.diff.DiffBasedAnomalyDetector:
+        gordo_components.machine.model.anomaly.diff.DiffBasedAnomalyDetector:
           base_estimator:
             sklearn.pipeline.Pipeline:
               steps:
                 - sklearn.preprocessing.data.MinMaxScaler
-                - gordo_components.model.models.KerasAutoEncoder:
+                - gordo_components.machine.model.models.KerasAutoEncoder:
                     kind: feedforward_hourglass
diff --git a/examples/config_legacy.yaml b/examples/config_legacy.yaml
@@ -6,19 +6,19 @@ machines:
   - name: ct-23-0001 #1st machine
     dataset:
       tags: #list of tags for 1st machine
-        - TAG 1
-        - TAG 2
-        - TAG 3
+        - GRA-TAG 1
+        - GRA-TAG 2
+        - GRA-TAG 3
       train_start_date: 2016-11-07T09:11:30+01:00
       train_end_date: 2018-09-15T03:01:00+01:00
 
   - name: ct-23-0002 #2nd machine
     dataset:
       resolution: 2T
       tags: #list of tags for 2nd machine
-        - TAG 1
-        - TAG 2
-        - TAG 3
+        - GRA-TAG 1
+        - GRA-TAG 2
+        - GRA-TAG 3
       train_start_date: 2018-05-20T01:00:04+02:00
       train_end_date: 2019-05-10T15:05:50+02:00
     runtime:
@@ -32,9 +32,9 @@ machines:
   - name: ct-23-0003 #3rd machine
     dataset:
       tags: #list of tags for 3rd machine
-        - TAG 1
-        - TAG 2
-        - TAG 3
+        - GRA-TAG 1
+        - GRA-TAG 2
+        - GRA-TAG 3
       train_start_date: 2018-09-15T13:03:04+02:00
       train_end_date: 2019-03-11T11:05:10+02:00
 
@@ -63,5 +63,5 @@ globals:
     sklearn.pipeline.Pipeline:
       steps:
         - sklearn.preprocessing.data.MinMaxScaler
-        - gordo_components.model.models.KerasAutoEncoder:
+        - gordo_components.machine.model.models.KerasAutoEncoder:
             kind: feedforward_hourglass