Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix Python linting #7060

Merged
merged 3 commits into from
Mar 30, 2023
Merged

Fix Python linting #7060

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
be7e954
gh-action: Workflow for testing python lint
kimwnasptd Mar 24, 2023
476af57
lint: Run autopep8 to lint files
kimwnasptd Mar 24, 2023
e3293df
linting: Manual changes
kimwnasptd Mar 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
linting: Manual changes 
Signed-off-by: Kimonas Sotirchos <kimwnasptd@arrikto.com>
  • Loading branch information
@kimwnasptd
kimwnasptd committed Mar 24, 2023
commit e3293df0a94f2881c678b3323477efacb6b79fe0
2 changes: 1 addition & 1 deletion components/crud-web-apps/common/backend/kubeflow/kubeflow/crud_backend/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def create_app(name, static_folder, config):
app.config.from_object(config)

if (config.ENV == BackendMode.DEVELOPMENT.value
or config.ENV == BackendMode.DEVELOPMENT_FULL.value):
or config.ENV == BackendMode.DEVELOPMENT_FULL.value): # noqa: W503
log.warn("RUNNING IN DEVELOPMENT MODE")

# Register all the blueprints
Expand Down
3 changes: 1 addition & 2 deletions components/crud-web-apps/common/backend/kubeflow/kubeflow/crud_backend/api/node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
from .. import authz
from . import custom_api, storage_api, v1_core
from . import v1_core


def list_nodes():
Expand Down
2 changes: 1 addition & 1 deletion components/crud-web-apps/common/backend/kubeflow/kubeflow/crud_backend/api/notebook.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from .. import authz
from . import custom_api, v1_core, utils, events
from . import custom_api, events, utils


def get_notebook(notebook, namespace):
Expand Down
3 changes: 2 additions & 1 deletion components/crud-web-apps/common/backend/kubeflow/kubeflow/crud_backend/api/pvc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ def get_pvc(pvc, namespace):

def list_pvc_events(namespace, pvc_name):

field_selector = utils.events_field_selector("PersistentVolumeClaim", pvc_name)
field_selector = utils.events_field_selector(
"PersistentVolumeClaim", pvc_name)

return events.list_events(namespace, field_selector)

Expand Down
2 changes: 1 addition & 1 deletion components/crud-web-apps/common/backend/kubeflow/kubeflow/crud_backend/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ class BackendMode(enum.Enum):

def dev_mode_enabled():
env = current_app.config.get("ENV")
return (env == BackendMode.DEVELOPMENT_FULL.value or
return (env == BackendMode.DEVELOPMENT_FULL.value or # noqa: W504
env == BackendMode.DEVELOPMENT.value)


Expand Down
3 changes: 2 additions & 1 deletion components/crud-web-apps/common/backend/kubeflow/kubeflow/crud_backend/csrf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@
"Strict".

References:
- OWASP CSRF Mitigation: https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html
- OWASP CSRF Mitigation:
https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html # noqa: E501
"""

import logging
Expand Down
6 changes: 3 additions & 3 deletions components/crud-web-apps/jupyter/backend/apps/common/form.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import json

from werkzeug.exceptions import BadRequest

from kubeflow.kubeflow.crud_backend import logging
from werkzeug.exceptions import BadRequest

from . import utils

Expand Down Expand Up @@ -82,7 +81,8 @@ def set_notebook_image(notebook, body, defaults):
image_body_field = "customImage"

image = get_form_value(body, defaults, image_body_field, "image")
notebook["spec"]["template"]["spec"]["containers"][0]["image"] = image.strip()
container = notebook["spec"]["template"]["spec"]["containers"][0]
container["image"] = image.strip()


def set_notebook_image_pull_policy(notebook, body, defaults):
Expand Down
4 changes: 1 addition & 3 deletions components/crud-web-apps/jupyter/backend/apps/common/routes/get.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
"""GET request handlers."""
from flask import request

from kubeflow.kubeflow.crud_backend import api, logging

from werkzeug.exceptions import NotFound

from .. import utils
Expand Down Expand Up @@ -82,7 +80,7 @@ def get_notebook_pod(notebook_name, namespace):
raise NotFound("No pod detected.")


@bp.route("/api/namespaces/<namespace>/notebooks/<notebook_name>/pod/<pod_name>/logs")
@bp.route("/api/namespaces/<namespace>/notebooks/<notebook_name>/pod/<pod_name>/logs") # noqa: E501
def get_pod_logs(namespace, notebook_name, pod_name):
container = notebook_name
logs = api.get_pod_logs(namespace, pod_name, container)
Expand Down
52 changes: 32 additions & 20 deletions components/crud-web-apps/jupyter/backend/apps/common/status.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@

def process_status(notebook):
"""
Return status and reason. Status may be [ready|waiting|warning|terminating|stopped]
Return status and reason. Status may be:
[ready|waiting|warning|terminating|stopped]
"""
# In case the Notebook has no status
status_phase, status_message = get_empty_status(notebook)
Expand All @@ -30,27 +31,29 @@ def process_status(notebook):
if status_phase is not None:
return status.create_status(status_phase, status_message)

# Extract information about the status from the containerState of the Notebook's status
# Extract information about the status from the containerState of the
# Notebook's status
status_phase, status_message = get_status_from_container_state(notebook)
if status_phase is not None:
return status.create_status(status_phase, status_message)

# Extract information about the status from the conditions of the Notebook's status
# Extract information about the status from the conditions of the
# Notebook's status
status_phase, status_message = get_status_from_conditions(notebook)
if status_phase is not None:
return status.create_status(status_phase, status_message)

# Try to extract information about why the notebook is not starting
# from the notebook's events (see find_error_event)
notebook_events = get_notebook_events(notebook)

# In case there no Events available, show a generic message
status_phase, status_message = status.STATUS_PHASE.WARNING, "Couldn't find any information for the status of this notebook."

status_event, reason_event = get_status_from_events(notebook_events)
if status_event is not None:
status_phase, status_message = status_event, reason_event

# In case there no Events available, show a generic message
status_phase = status.STATUS_PHASE.WARNING
status_message = "Couldn't find any information for the status of this notebook." # noqa: E501

return status.create_status(status_phase, status_message)


Expand All @@ -65,10 +68,12 @@ def get_empty_status(notebook):
current_time = dt.datetime.utcnow().replace(microsecond=0)
delta = (current_time - nb_creation_time)

# If the Notebook has no status, the status will be waiting (instead of warning) and we will
# show a generic message for the first 10 seconds
# If the Notebook has no status, the status will be waiting
# (instead of warning) and we will show a generic message for the first 10
# seconds
if not container_state and not conditions and delta.total_seconds() <= 10:
status_phase, status_message = status.STATUS_PHASE.WAITING, "Waiting for StatefulSet to create the underlying Pod."
status_phase = status.STATUS_PHASE.WAITING
status_message = "Waiting for StatefulSet to create the underlying Pod." # noqa: E501
return status_phase, status_message

return None, None
Expand All @@ -82,11 +87,13 @@ def get_stopped_status(notebook):
if STOP_ANNOTATION in annotations:
# If the Notebook is stopped, the status will be stopped
if ready_replicas == 0:
status_phase, status_message = status.STATUS_PHASE.STOPPED, "No Pods are currently running for this Notebook Server."
status_phase = status.STATUS_PHASE.STOPPED
status_message = "No Pods are currently running for this Notebook Server." # noqa: E501
return status_phase, status_message
# If the Notebook is being stopped, the status will be waiting
else:
status_phase, status_message = status.STATUS_PHASE.WAITING, "Notebook Server is stopping."
status_phase = status.STATUS_PHASE.WAITING
status_message = "Notebook Server is stopping."
return status_phase, status_message

return None, None
Expand All @@ -97,7 +104,8 @@ def get_deleted_status(notebook):

# If the Notebook is being deleted, the status will be terminating
if "deletionTimestamp" in metadata:
status_phase, status_message = status.STATUS_PHASE.TERMINATING, "Deleting this Notebook Server."
status_phase = status.STATUS_PHASE.TERMINATING
status_message = "Deleting this Notebook Server."
return status_phase, status_message

return None, None
Expand All @@ -120,13 +128,17 @@ def get_status_from_container_state(notebook):
if "waiting" in container_state:
# If the Notebook is initializing, the status will be waiting
if container_state["waiting"]["reason"] == 'PodInitializing':
status_phase, status_message = status.STATUS_PHASE.WAITING, container_state[
"waiting"]["reason"]
status_phase = status.STATUS_PHASE.WAITING
status_message = container_state["waiting"]["reason"]
return status_phase, status_message
# In any other case, the status will be warning with a "reason: message" showing on hover
# In any other case, the status will be warning with a "reason:
# message" showing on hover
else:
status_phase, status_message = status.STATUS_PHASE.WARNING, container_state[
"waiting"]["reason"] + ': ' + container_state["waiting"]["message"]
status_phase = status.STATUS_PHASE.WARNING

reason = container_state["waiting"]["reason"]
message = container_state["waiting"]["message"]
status_message = '%s: %s' % (reason, message)
return status_phase, status_message

return None, None
Expand All @@ -138,8 +150,8 @@ def get_status_from_conditions(notebook):
for condition in conditions:
# The status will be warning with a "reason: message" showing on hover
if "reason" in condition:
status_phase, status_message = status.STATUS_PHASE.WARNING, condition[
"reason"] + ': ' + condition["message"]
status_phase = status.STATUS_PHASE.WARNING
status_message = condition["reason"] + ': ' + condition["message"]
return status_phase, status_message

return None, None
Expand Down
2 changes: 1 addition & 1 deletion components/crud-web-apps/volumes/backend/apps/common/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from kubeflow.kubeflow.crud_backend import api, helpers
from kubeflow.kubeflow.crud_backend import api

from . import status

Expand Down
13 changes: 7 additions & 6 deletions components/notebook-controller/loadtest/start_notebooks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# This script aims to load test Kubeflow Notebook controller by starting
# certain nubmer of Kubeflow Notebook custom resources.
#
# Before the test, make sure you have connected to your desired Kubeflow cluster
# and have enough Kubernetes resources (or have autoscaling turned on).
# Before the test, make sure you have connected to your desired Kubeflow
# cluster and have enough Kubernetes resources (or have autoscaling turned on).
#
# To start the load test, you can run
# python3.8 start_notebooks.py -l <#notebooks> -n <namespace>
Expand All @@ -12,6 +12,7 @@

import argparse
import subprocess

import yaml

parser = argparse.ArgumentParser(
Expand Down Expand Up @@ -49,10 +50,10 @@

def write_notebook_config(config, name, num):
config['metadata']['name'] = 'jupyter-test-' + str(num)
config['spec']['template']['spec']['containers'][0]['name'
] = 'notebook-' + str(num)
config['spec']['template']['spec']['volumes'][0]['persistentVolumeClaim'][
'claimName'] = 'test-vol-' + str(num)
spec = config['spec']['template']['spec']
spec['containers'][0]['name'] = 'notebook-' + str(num)
pvc = spec['volumes'][0]['persistentVolumeClaim']
pvc['claimName'] = 'test-vol-' + str(num)
with open(name, 'w') as f:
print(yaml.dump(config), file=f)

Expand Down
12 changes: 6 additions & 6 deletions components/notebook-controller/third_party/concatenate_license.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,26 +11,26 @@
# limitations under the License.

import argparse
import requests
import sys
import traceback

import requests

parser = argparse.ArgumentParser(
description='Generate dependencies json from license.csv file.')
parser.add_argument(
'license_info_file',
nargs='?',
default='license_info.csv',
help='CSV file with license info fetched from github using get-github-license-info CLI tool.'
+ '(default: %(default)s)',
help='CSV file with license info fetched from github using get-github-license-info CLI tool.' + '(default: %(default)s)', # noqa: E501
)
parser.add_argument(
'-o',
'--output',
dest='output_file',
nargs='?',
default='license.txt',
help='Concatenated license file path this command generates. (default: %(default)s)'
help='Concatenated license file path this command generates. (default: %(default)s)' # noqa: E501
)
args = parser.parse_args()

Expand All @@ -57,13 +57,13 @@ def main():
file=sys.stderr)
license_text = fetch_license_text(license_download_link)
print(
'--------------------------------------------------------------------------------',
'--------------------------------------------------------',
file=output_file,
)
print('{} {} {}'.format(repo, license_name, license_link),
file=output_file)
print(
'--------------------------------------------------------------------------------',
'--------------------------------------------------------',
file=output_file,
)
print(license_text, file=output_file)
Expand Down
5 changes: 3 additions & 2 deletions py/kubeflow/kubeflow/cd/base_runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import os
import yaml
from importlib import import_module

import yaml


def main(component_name, workflow_name):
component = import_module("kubeflow.kubeflow.cd.%s" % component_name)
Expand All @@ -16,4 +17,4 @@ def main(component_name, workflow_name):


if __name__ == "__main__":
main(component, workflow_name)
main(component, workflow_name) # noqa: F821
24 changes: 12 additions & 12 deletions py/kubeflow/kubeflow/cd/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@
NOTEBOOK_SERVER_JUPYTER = "%s/notebook-servers/jupyter" % AWS_REGISTRY
NOTEBOOK_SERVER_RSTUDIO = "%s/notebook-servers/rstudio" % AWS_REGISTRY
NOTEBOOK_SERVER_CODESERVER = "%s/notebook-servers/codeserver" % AWS_REGISTRY
NOTEBOOK_SERVER_JUPYTER_PYTORCH = "%s/notebook-servers/jupyter-pytorch" % AWS_REGISTRY
NOTEBOOK_SERVER_JUPYTER_PYTORCH_CUDA = "%s/notebook-servers/jupyter-pytorch-cuda" % AWS_REGISTRY
NOTEBOOK_SERVER_JUPYTER_TENSORFLOW = "%s/notebook-servers/jupyter-tensorflow" % AWS_REGISTRY
NOTEBOOK_SERVER_JUPYTER_TENSORFLOW_CUDA = "%s/notebook-servers/jupyter-tensorflow-cuda" % AWS_REGISTRY
NOTEBOOK_SERVER_JUPYTER_PYTORCH_FULL = "%s/notebook-servers/jupyter-pytorch-full" % AWS_REGISTRY
NOTEBOOK_SERVER_JUPYTER_PYTORCH_CUDA_FULL = "%s/notebook-servers/jupyter-pytorch-cuda-full" % AWS_REGISTRY
NOTEBOOK_SERVER_JUPYTER_TENSORFLOW_FULL = "%s/notebook-servers/jupyter-tensorflow-full" % AWS_REGISTRY
NOTEBOOK_SERVER_JUPYTER_TENSORFLOW_CUDA_FULL = "%s/notebook-servers/jupyter-tensorflow-cuda-full" % AWS_REGISTRY
NOTEBOOK_SERVER_JUPYTER_SCIPY = "%s/notebook-servers/jupyter-scipy" % AWS_REGISTRY
NOTEBOOK_SERVER_CODESERVER_PYTHON = "%s/notebook-servers/codeserver-python" % AWS_REGISTRY
NOTEBOOK_SERVER_RSTUDIO_TIDYVERSE = "%s/notebook-servers/rstudio-tidyverse" % AWS_REGISTRY
NOTEBOOK_SERVER_JUPYTER_PYSPARK = "%s/notebook-servers/jupyter-pyspark" % AWS_REGISTRY
NOTEBOOK_SERVER_JUPYTER_PYTORCH = "%s/notebook-servers/jupyter-pytorch" % AWS_REGISTRY # noqa: E501
NOTEBOOK_SERVER_JUPYTER_PYTORCH_CUDA = "%s/notebook-servers/jupyter-pytorch-cuda" % AWS_REGISTRY # noqa: E501
NOTEBOOK_SERVER_JUPYTER_TENSORFLOW = "%s/notebook-servers/jupyter-tensorflow" % AWS_REGISTRY # noqa: E501
NOTEBOOK_SERVER_JUPYTER_TENSORFLOW_CUDA = "%s/notebook-servers/jupyter-tensorflow-cuda" % AWS_REGISTRY # noqa: E501
NOTEBOOK_SERVER_JUPYTER_PYTORCH_FULL = "%s/notebook-servers/jupyter-pytorch-full" % AWS_REGISTRY # noqa: E501
NOTEBOOK_SERVER_JUPYTER_PYTORCH_CUDA_FULL = "%s/notebook-servers/jupyter-pytorch-cuda-full" % AWS_REGISTRY # noqa: E501
NOTEBOOK_SERVER_JUPYTER_TENSORFLOW_FULL = "%s/notebook-servers/jupyter-tensorflow-full" % AWS_REGISTRY # noqa: E501
NOTEBOOK_SERVER_JUPYTER_TENSORFLOW_CUDA_FULL = "%s/notebook-servers/jupyter-tensorflow-cuda-full" % AWS_REGISTRY # noqa: E501
NOTEBOOK_SERVER_JUPYTER_SCIPY = "%s/notebook-servers/jupyter-scipy" % AWS_REGISTRY # noqa: E501
NOTEBOOK_SERVER_CODESERVER_PYTHON = "%s/notebook-servers/codeserver-python" % AWS_REGISTRY # noqa: E501
NOTEBOOK_SERVER_RSTUDIO_TIDYVERSE = "%s/notebook-servers/rstudio-tidyverse" % AWS_REGISTRY # noqa: E501
NOTEBOOK_SERVER_JUPYTER_PYSPARK = "%s/notebook-servers/jupyter-pyspark" % AWS_REGISTRY # noqa: E501
4 changes: 3 additions & 1 deletion py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_codeserver.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
""""Argo Workflow for building notebook-server-codeserver's OCI image using Kaniko"""
""""
Argo Workflow for building notebook-server-codeserver's OCI image using Kaniko.
"""
from kubeflow.kubeflow.cd import config, kaniko_builder


Expand Down
7 changes: 5 additions & 2 deletions py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_codeserver_python.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
""""Argo Workflow for building notebook-server-codeserver-python's OCI image using Kaniko"""
""""
Argo Workflow for building notebook-server-codeserver-python's OCI image
using Kaniko.
"""
from kubeflow.kubeflow.cd import config, kaniko_builder


Expand All @@ -15,6 +18,6 @@ def create_workflow(name=None, namespace=None, bucket=None, **kwargs):
**kwargs)

return builder.build(
dockerfile="components/example-notebook-servers/codeserver-python/Dockerfile",
dockerfile="components/example-notebook-servers/codeserver-python/Dockerfile", # noqa: E501
context="components/example-notebook-servers/codeserver-python/",
destination=config.NOTEBOOK_SERVER_CODESERVER_PYTHON)
2 changes: 1 addition & 1 deletion py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_jupyter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
""""Argo Workflow for building notebook-server-jupyter's OCI image using Kaniko"""
""""Argo Workflow for building notebook-server-jupyter's image using Kaniko"""
from kubeflow.kubeflow.cd import config, kaniko_builder


Expand Down
9 changes: 6 additions & 3 deletions py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_jupyter_pytorch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
""""Argo Workflow for building the notebook-server-jupyter-pytorch OCI images using Kaniko"""
""""
Argo Workflow for building the notebook-server-jupyter-pytorch OCI images using
Kaniko.
"""
from kubeflow.kubeflow.cd import config, kaniko_builder


Expand All @@ -15,10 +18,10 @@ def create_workflow(name=None, namespace=None, bucket=None, **kwargs):
**kwargs)

return builder.build(
dockerfile="components/example-notebook-servers/jupyter-pytorch/cpu.Dockerfile",
dockerfile="components/example-notebook-servers/jupyter-pytorch/cpu.Dockerfile", # noqa: E501
context="components/example-notebook-servers/jupyter-pytorch/",
destination=config.NOTEBOOK_SERVER_JUPYTER_PYTORCH,
second_dockerfile="components/example-notebook-servers/jupyter-pytorch/cuda.Dockerfile",
second_dockerfile="components/example-notebook-servers/jupyter-pytorch/cuda.Dockerfile", # noqa: E501
second_destination=config.NOTEBOOK_SERVER_JUPYTER_PYTORCH_CUDA,
mem_override="8Gi",
deadline_override=6000)
Loading