Skip to content

Commit

Permalink
Feat: add labels to all resource creation apis (#601)
Browse files Browse the repository at this point in the history 
Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly:
- [x] Ensure the tests and linter pass
- [x] Code coverage does not decrease (if any source code was changed)
- [x] Appropriate docs were updated (if necessary)

Fixes #595, <b/191789899> 🦕

- Add support for labels in resources creation: 
    - datasets (`create`)
        - ImageDataset / TabularDataset / TextDataset / TimeSeriesDataset / VideoDataset
    - tensorboard 
        - `Tensorboard.create`
    - jobs
        - `BatchPredictionJob.create`
        - `CustomJob.__init__`
        - `HyperparameterTuningJob.__init__`
    - models
        - `Endpoint.create`
        - `Model.upload` / `Model.batch_predict`
    - training_jobs (`__init__` for training labels, `run` for model labels)
        - CustomTrainingJob / CustomContainerTrainingJob / CustomPythonPackageTrainingJob 
        - AutoMLTabularTrainingJob / AutoMLForecastingTrainingJob / AutoMLImageTrainingJob / AutoMLVideoTrainingJob / AutoMLTextTrainingJob 

- Modify `pipeline_jobs.py` and `utils.py` for `validate_labels` reusability
- Add / modify unit tests to verify:
    - standard resource creation with / without labels
    - training pipeline resource creation
        - when model labels is not provided
        - when base model labels is provided
  • Loading branch information
@morgandu
morgandu authored Aug 11, 2021
1 parent b478075 commit 4e7666a
Show file tree
Hide file tree
Showing 24 changed files with 872 additions and 64 deletions.
40 changes: 39 additions & 1 deletion google/cloud/aiplatform/datasets/dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
#

from typing import Optional, Sequence, Dict, Tuple, Union, List
from typing import Dict, List, Optional, Sequence, Tuple, Union

from google.api_core import operation
from google.auth import credentials as auth_credentials
Expand Down Expand Up @@ -115,6 +115,7 @@ def create(
location: Optional[str] = None,
credentials: Optional[auth_credentials.Credentials] = None,
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
labels: Optional[Dict[str, str]] = None,
encryption_spec_key_name: Optional[str] = None,
sync: bool = True,
) -> "_Dataset":
Expand Down Expand Up @@ -176,6 +177,16 @@ def create(
credentials set in aiplatform.init.
request_metadata (Sequence[Tuple[str, str]]):
Strings which should be sent along with the request as metadata.
labels (Dict[str, str]):
Optional. Labels with user-defined metadata to organize your Tensorboards.
Label keys and values can be no longer than 64 characters
(Unicode codepoints), can only contain lowercase letters, numeric
characters, underscores and dashes. International characters are allowed.
No more than 64 user labels can be associated with one Tensorboard
(System labels are excluded).
See https://goo.gl/xmQnxf for more information and examples of labels.
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
and are immutable.
encryption_spec_key_name (Optional[str]):
Optional. The Cloud KMS resource identifier of the customer
managed encryption key used to protect the dataset. Has the
Expand All @@ -198,6 +209,8 @@ def create(
"""

utils.validate_display_name(display_name)
if labels:
utils.validate_labels(labels)

api_client = cls._instantiate_client(location=location, credentials=credentials)

Expand All @@ -221,6 +234,7 @@ def create(
location=location or initializer.global_config.location,
credentials=credentials or initializer.global_config.credentials,
request_metadata=request_metadata,
labels=labels,
encryption_spec=initializer.global_config.get_encryption_spec(
encryption_spec_key_name=encryption_spec_key_name
),
Expand All @@ -240,6 +254,7 @@ def _create_and_import(
location: str,
credentials: Optional[auth_credentials.Credentials],
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
labels: Optional[Dict[str, str]] = None,
encryption_spec: Optional[gca_encryption_spec.EncryptionSpec] = None,
sync: bool = True,
) -> "_Dataset":
Expand Down Expand Up @@ -277,6 +292,16 @@ def _create_and_import(
credentials set in aiplatform.init.
request_metadata (Sequence[Tuple[str, str]]):
Strings which should be sent along with the request as metadata.
labels (Dict[str, str]):
Optional. Labels with user-defined metadata to organize your Tensorboards.
Label keys and values can be no longer than 64 characters
(Unicode codepoints), can only contain lowercase letters, numeric
characters, underscores and dashes. International characters are allowed.
No more than 64 user labels can be associated with one Tensorboard
(System labels are excluded).
See https://goo.gl/xmQnxf for more information and examples of labels.
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
and are immutable.
encryption_spec (Optional[gca_encryption_spec.EncryptionSpec]):
Optional. The Cloud KMS customer managed encryption key used to protect the dataset.
The key needs to be in the same region as where the compute
Expand All @@ -300,6 +325,7 @@ def _create_and_import(
metadata_schema_uri=metadata_schema_uri,
datasource=datasource,
request_metadata=request_metadata,
labels=labels,
encryption_spec=encryption_spec,
)

Expand Down Expand Up @@ -346,6 +372,7 @@ def _create(
metadata_schema_uri: str,
datasource: _datasources.Datasource,
request_metadata: Sequence[Tuple[str, str]] = (),
labels: Optional[Dict[str, str]] = None,
encryption_spec: Optional[gca_encryption_spec.EncryptionSpec] = None,
) -> operation.Operation:
"""Creates a new managed dataset by directly calling API client.
Expand Down Expand Up @@ -373,6 +400,16 @@ def _create(
request_metadata (Sequence[Tuple[str, str]]):
Strings which should be sent along with the create_dataset
request as metadata. Usually to specify special dataset config.
labels (Dict[str, str]):
Optional. Labels with user-defined metadata to organize your Tensorboards.
Label keys and values can be no longer than 64 characters
(Unicode codepoints), can only contain lowercase letters, numeric
characters, underscores and dashes. International characters are allowed.
No more than 64 user labels can be associated with one Tensorboard
(System labels are excluded).
See https://goo.gl/xmQnxf for more information and examples of labels.
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
and are immutable.
encryption_spec (Optional[gca_encryption_spec.EncryptionSpec]):
Optional. The Cloud KMS customer managed encryption key used to protect the dataset.
The key needs to be in the same region as where the compute
Expand All @@ -388,6 +425,7 @@ def _create(
display_name=display_name,
metadata_schema_uri=metadata_schema_uri,
metadata=datasource.dataset_metadata,
labels=labels,
encryption_spec=encryption_spec,
)

Expand Down
16 changes: 15 additions & 1 deletion google/cloud/aiplatform/datasets/image_dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
#

from typing import Optional, Sequence, Dict, Tuple, Union
from typing import Dict, Optional, Sequence, Tuple, Union

from google.auth import credentials as auth_credentials

Expand Down Expand Up @@ -44,6 +44,7 @@ def create(
location: Optional[str] = None,
credentials: Optional[auth_credentials.Credentials] = None,
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
labels: Optional[Dict[str, str]] = None,
encryption_spec_key_name: Optional[str] = None,
sync: bool = True,
) -> "ImageDataset":
Expand Down Expand Up @@ -95,6 +96,16 @@ def create(
credentials set in aiplatform.init.
request_metadata (Sequence[Tuple[str, str]]):
Strings which should be sent along with the request as metadata.
labels (Dict[str, str]):
Optional. Labels with user-defined metadata to organize your Tensorboards.
Label keys and values can be no longer than 64 characters
(Unicode codepoints), can only contain lowercase letters, numeric
characters, underscores and dashes. International characters are allowed.
No more than 64 user labels can be associated with one Tensorboard
(System labels are excluded).
See https://goo.gl/xmQnxf for more information and examples of labels.
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
and are immutable.
encryption_spec_key_name (Optional[str]):
Optional. The Cloud KMS resource identifier of the customer
managed encryption key used to protect the dataset. Has the
Expand All @@ -117,6 +128,8 @@ def create(
"""

utils.validate_display_name(display_name)
if labels:
utils.validate_labels(labels)

api_client = cls._instantiate_client(location=location, credentials=credentials)

Expand All @@ -141,6 +154,7 @@ def create(
location=location or initializer.global_config.location,
credentials=credentials or initializer.global_config.credentials,
request_metadata=request_metadata,
labels=labels,
encryption_spec=initializer.global_config.get_encryption_spec(
encryption_spec_key_name=encryption_spec_key_name
),
Expand Down
16 changes: 15 additions & 1 deletion google/cloud/aiplatform/datasets/tabular_dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import csv
import logging

from typing import List, Optional, Sequence, Set, Tuple, Union
from typing import Dict, List, Optional, Sequence, Set, Tuple, Union

from google.auth import credentials as auth_credentials

Expand Down Expand Up @@ -269,6 +269,7 @@ def create(
location: Optional[str] = None,
credentials: Optional[auth_credentials.Credentials] = None,
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
labels: Optional[Dict[str, str]] = None,
encryption_spec_key_name: Optional[str] = None,
sync: bool = True,
) -> "TabularDataset":
Expand Down Expand Up @@ -302,6 +303,16 @@ def create(
credentials set in aiplatform.init.
request_metadata (Sequence[Tuple[str, str]]):
Strings which should be sent along with the request as metadata.
labels (Dict[str, str]):
Optional. Labels with user-defined metadata to organize your Tensorboards.
Label keys and values can be no longer than 64 characters
(Unicode codepoints), can only contain lowercase letters, numeric
characters, underscores and dashes. International characters are allowed.
No more than 64 user labels can be associated with one Tensorboard
(System labels are excluded).
See https://goo.gl/xmQnxf for more information and examples of labels.
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
and are immutable.
encryption_spec_key_name (Optional[str]):
Optional. The Cloud KMS resource identifier of the customer
managed encryption key used to protect the dataset. Has the
Expand All @@ -324,6 +335,8 @@ def create(
"""

utils.validate_display_name(display_name)
if labels:
utils.validate_labels(labels)

api_client = cls._instantiate_client(location=location, credentials=credentials)

Expand All @@ -347,6 +360,7 @@ def create(
location=location or initializer.global_config.location,
credentials=credentials or initializer.global_config.credentials,
request_metadata=request_metadata,
labels=labels,
encryption_spec=initializer.global_config.get_encryption_spec(
encryption_spec_key_name=encryption_spec_key_name
),
Expand Down
16 changes: 15 additions & 1 deletion google/cloud/aiplatform/datasets/text_dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
#

from typing import Optional, Sequence, Dict, Tuple, Union
from typing import Dict, Optional, Sequence, Tuple, Union

from google.auth import credentials as auth_credentials

Expand Down Expand Up @@ -44,6 +44,7 @@ def create(
location: Optional[str] = None,
credentials: Optional[auth_credentials.Credentials] = None,
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
labels: Optional[Dict[str, str]] = None,
encryption_spec_key_name: Optional[str] = None,
sync: bool = True,
) -> "TextDataset":
Expand Down Expand Up @@ -102,6 +103,16 @@ def create(
credentials set in aiplatform.init.
request_metadata (Sequence[Tuple[str, str]]):
Strings which should be sent along with the request as metadata.
labels (Dict[str, str]):
Optional. Labels with user-defined metadata to organize your Tensorboards.
Label keys and values can be no longer than 64 characters
(Unicode codepoints), can only contain lowercase letters, numeric
characters, underscores and dashes. International characters are allowed.
No more than 64 user labels can be associated with one Tensorboard
(System labels are excluded).
See https://goo.gl/xmQnxf for more information and examples of labels.
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
and are immutable.
encryption_spec_key_name (Optional[str]):
Optional. The Cloud KMS resource identifier of the customer
managed encryption key used to protect the dataset. Has the
Expand All @@ -124,6 +135,8 @@ def create(
"""

utils.validate_display_name(display_name)
if labels:
utils.validate_labels(labels)

api_client = cls._instantiate_client(location=location, credentials=credentials)

Expand All @@ -148,6 +161,7 @@ def create(
location=location or initializer.global_config.location,
credentials=credentials or initializer.global_config.credentials,
request_metadata=request_metadata,
labels=labels,
encryption_spec=initializer.global_config.get_encryption_spec(
encryption_spec_key_name=encryption_spec_key_name
),
Expand Down
16 changes: 15 additions & 1 deletion google/cloud/aiplatform/datasets/time_series_dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
#

from typing import Optional, Sequence, Tuple, Union
from typing import Dict, Optional, Sequence, Tuple, Union

from google.auth import credentials as auth_credentials

Expand Down Expand Up @@ -43,6 +43,7 @@ def create(
location: Optional[str] = None,
credentials: Optional[auth_credentials.Credentials] = None,
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
labels: Optional[Dict[str, str]] = None,
encryption_spec_key_name: Optional[str] = None,
sync: bool = True,
) -> "TimeSeriesDataset":
Expand Down Expand Up @@ -76,6 +77,16 @@ def create(
credentials set in aiplatform.init.
request_metadata (Sequence[Tuple[str, str]]):
Strings which should be sent along with the request as metadata.
labels (Dict[str, str]):
Optional. Labels with user-defined metadata to organize your Tensorboards.
Label keys and values can be no longer than 64 characters
(Unicode codepoints), can only contain lowercase letters, numeric
characters, underscores and dashes. International characters are allowed.
No more than 64 user labels can be associated with one Tensorboard
(System labels are excluded).
See https://goo.gl/xmQnxf for more information and examples of labels.
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
and are immutable.
encryption_spec_key_name (Optional[str]):
Optional. The Cloud KMS resource identifier of the customer
managed encryption key used to protect the dataset. Has the
Expand All @@ -99,6 +110,8 @@ def create(
"""

utils.validate_display_name(display_name)
if labels:
utils.validate_labels(labels)

api_client = cls._instantiate_client(location=location, credentials=credentials)

Expand All @@ -122,6 +135,7 @@ def create(
location=location or initializer.global_config.location,
credentials=credentials or initializer.global_config.credentials,
request_metadata=request_metadata,
labels=labels,
encryption_spec=initializer.global_config.get_encryption_spec(
encryption_spec_key_name=encryption_spec_key_name
),
Expand Down
16 changes: 15 additions & 1 deletion google/cloud/aiplatform/datasets/video_dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.
#

from typing import Optional, Sequence, Dict, Tuple, Union
from typing import Dict, Optional, Sequence, Tuple, Union

from google.auth import credentials as auth_credentials

Expand Down Expand Up @@ -44,6 +44,7 @@ def create(
location: Optional[str] = None,
credentials: Optional[auth_credentials.Credentials] = None,
request_metadata: Optional[Sequence[Tuple[str, str]]] = (),
labels: Optional[Dict[str, str]] = None,
encryption_spec_key_name: Optional[str] = None,
sync: bool = True,
) -> "VideoDataset":
Expand Down Expand Up @@ -95,6 +96,16 @@ def create(
credentials set in aiplatform.init.
request_metadata (Sequence[Tuple[str, str]]):
Strings which should be sent along with the request as metadata.
labels (Dict[str, str]):
Optional. Labels with user-defined metadata to organize your Tensorboards.
Label keys and values can be no longer than 64 characters
(Unicode codepoints), can only contain lowercase letters, numeric
characters, underscores and dashes. International characters are allowed.
No more than 64 user labels can be associated with one Tensorboard
(System labels are excluded).
See https://goo.gl/xmQnxf for more information and examples of labels.
System reserved label keys are prefixed with "aiplatform.googleapis.com/"
and are immutable.
encryption_spec_key_name (Optional[str]):
Optional. The Cloud KMS resource identifier of the customer
managed encryption key used to protect the dataset. Has the
Expand All @@ -117,6 +128,8 @@ def create(
"""

utils.validate_display_name(display_name)
if labels:
utils.validate_labels(labels)

api_client = cls._instantiate_client(location=location, credentials=credentials)

Expand All @@ -141,6 +154,7 @@ def create(
location=location or initializer.global_config.location,
credentials=credentials or initializer.global_config.credentials,
request_metadata=request_metadata,
labels=labels,
encryption_spec=initializer.global_config.get_encryption_spec(
encryption_spec_key_name=encryption_spec_key_name
),
Expand Down
Loading

0 comments on commit 4e7666a

Please sign in to comment.