diff --git a/utils/__main__.py b/src/dvtests/__main__.py similarity index 84% rename from utils/__main__.py rename to src/dvtests/__main__.py index 3bcb373..e8b822e 100644 --- a/utils/__main__.py +++ b/src/dvtests/__main__.py @@ -2,10 +2,7 @@ from typing import List import typer -from config import Config - from utils import collect_data -from utils import config from utils import create_testdata from utils import generate_data from utils import INSTANCE_DATA_DIR @@ -41,13 +38,12 @@ def create_testdata_command(config_file: str, force: bool = False) -> None: @app.command("remove-testdata") def remove_testdata_command( - config_file: str = None, - parent: str = None, + parent: str, + parent_data_type: str = "dataverse", data_types: List[str] = ["dataverses", "datasets"], - ds_published: bool = False, force: bool = False, ) -> None: - remove_testdata(config_file, parent, data_types, ds_published, force) + remove_testdata(parent, parent_data_type, data_types, force) typer.echo(f"Testdata removed") diff --git a/testing/__init__.py b/src/dvtests/testing/__init__.py similarity index 100% rename from testing/__init__.py rename to src/dvtests/testing/__init__.py diff --git a/testing/config.py b/src/dvtests/testing/config.py similarity index 100% rename from testing/config.py rename to src/dvtests/testing/config.py diff --git a/testing/conftest.py b/src/dvtests/testing/conftest.py similarity index 100% rename from testing/conftest.py rename to src/dvtests/testing/conftest.py diff --git a/testing/data/dataverse_test-upload.json b/src/dvtests/testing/data/dataverse_test-upload.json similarity index 100% rename from testing/data/dataverse_test-upload.json rename to src/dvtests/testing/data/dataverse_test-upload.json diff --git a/testing/data/instances/dataverse_dv03/test-config.json b/src/dvtests/testing/data/instances/dataverse_dv03/test-config.json similarity index 100% rename from testing/data/instances/dataverse_dv03/test-config.json rename to src/dvtests/testing/data/instances/dataverse_dv03/test-config.json diff --git a/testing/data/instances/dataverse_dv05/test-config.json b/src/dvtests/testing/data/instances/dataverse_dv05/test-config.json similarity index 100% rename from testing/data/instances/dataverse_dv05/test-config.json rename to src/dvtests/testing/data/instances/dataverse_dv05/test-config.json diff --git a/testing/data/instances/dataverse_localhost_t550/test-config.json b/src/dvtests/testing/data/instances/dataverse_localhost_t550/test-config.json similarity index 100% rename from testing/data/instances/dataverse_localhost_t550/test-config.json rename to src/dvtests/testing/data/instances/dataverse_localhost_t550/test-config.json diff --git a/testing/data/instances/dataverse_production/test-config.json b/src/dvtests/testing/data/instances/dataverse_production/test-config.json similarity index 100% rename from testing/data/instances/dataverse_production/test-config.json rename to src/dvtests/testing/data/instances/dataverse_production/test-config.json diff --git a/testing/data/instances/website/test-data.json b/src/dvtests/testing/data/instances/website/test-data.json similarity index 100% rename from testing/data/instances/website/test-data.json rename to src/dvtests/testing/data/instances/website/test-data.json diff --git a/testing/dataverse/__init__.py b/src/dvtests/testing/dataverse/__init__.py similarity index 100% rename from testing/dataverse/__init__.py rename to src/dvtests/testing/dataverse/__init__.py diff --git a/testing/dataverse/suite_basic.py b/src/dvtests/testing/dataverse/suite_basic.py similarity index 100% rename from testing/dataverse/suite_basic.py rename to src/dvtests/testing/dataverse/suite_basic.py diff --git a/testing/dataverse/suite_data.py b/src/dvtests/testing/dataverse/suite_data.py similarity index 100% rename from testing/dataverse/suite_data.py rename to src/dvtests/testing/dataverse/suite_data.py diff --git a/testing/dataverse/test_api.py b/src/dvtests/testing/dataverse/test_api.py similarity index 100% rename from testing/dataverse/test_api.py rename to src/dvtests/testing/dataverse/test_api.py diff --git a/testing/dataverse/test_datafiles.py b/src/dvtests/testing/dataverse/test_datafiles.py similarity index 100% rename from testing/dataverse/test_datafiles.py rename to src/dvtests/testing/dataverse/test_datafiles.py diff --git a/testing/dataverse/test_datasets.py b/src/dvtests/testing/dataverse/test_datasets.py similarity index 100% rename from testing/dataverse/test_datasets.py rename to src/dvtests/testing/dataverse/test_datasets.py diff --git a/testing/dataverse/test_dataverses.py b/src/dvtests/testing/dataverse/test_dataverses.py similarity index 100% rename from testing/dataverse/test_dataverses.py rename to src/dvtests/testing/dataverse/test_dataverses.py diff --git a/testing/dataverse/test_homepage.py b/src/dvtests/testing/dataverse/test_homepage.py similarity index 100% rename from testing/dataverse/test_homepage.py rename to src/dvtests/testing/dataverse/test_homepage.py diff --git a/testing/dataverse/test_metadata_server.py b/src/dvtests/testing/dataverse/test_metadata_server.py similarity index 100% rename from testing/dataverse/test_metadata_server.py rename to src/dvtests/testing/dataverse/test_metadata_server.py diff --git a/testing/dataverse/test_resources.py b/src/dvtests/testing/dataverse/test_resources.py similarity index 100% rename from testing/dataverse/test_resources.py rename to src/dvtests/testing/dataverse/test_resources.py diff --git a/testing/dataverse/test_robots_txt.py b/src/dvtests/testing/dataverse/test_robots_txt.py similarity index 100% rename from testing/dataverse/test_robots_txt.py rename to src/dvtests/testing/dataverse/test_robots_txt.py diff --git a/testing/dataverse/test_search.py b/src/dvtests/testing/dataverse/test_search.py similarity index 100% rename from testing/dataverse/test_search.py rename to src/dvtests/testing/dataverse/test_search.py diff --git a/testing/dataverse/test_sitemap.py b/src/dvtests/testing/dataverse/test_sitemap.py similarity index 100% rename from testing/dataverse/test_sitemap.py rename to src/dvtests/testing/dataverse/test_sitemap.py diff --git a/testing/dataverse/test_user_authentication.py b/src/dvtests/testing/dataverse/test_user_authentication.py similarity index 100% rename from testing/dataverse/test_user_authentication.py rename to src/dvtests/testing/dataverse/test_user_authentication.py diff --git a/testing/dataverse/test_user_profile.py b/src/dvtests/testing/dataverse/test_user_profile.py similarity index 100% rename from testing/dataverse/test_user_profile.py rename to src/dvtests/testing/dataverse/test_user_profile.py diff --git a/testing/website/__init__.py b/src/dvtests/testing/website/__init__.py similarity index 100% rename from testing/website/__init__.py rename to src/dvtests/testing/website/__init__.py diff --git a/testing/website/test_resources.py b/src/dvtests/testing/website/test_resources.py similarity index 100% rename from testing/website/test_resources.py rename to src/dvtests/testing/website/test_resources.py diff --git a/utils/utils.py b/src/dvtests/utils/__init__.py similarity index 63% rename from utils/utils.py rename to src/dvtests/utils/__init__.py index d2fc779..02b3868 100644 --- a/utils/utils.py +++ b/src/dvtests/utils/__init__.py @@ -8,7 +8,6 @@ from time import sleep from typing import List -from config import Config from pyDataverse.api import NativeApi from pyDataverse.models import Datafile from pyDataverse.models import Dataset @@ -18,6 +17,8 @@ from pyDataverse.utils import read_json from pyDataverse.utils import write_json +from .config import Config + if os.getenv("ENV_FILE"): config = Config(_env_file=os.getenv("ENV_FILE")) @@ -29,7 +30,9 @@ ) if not os.path.isdir(INSTANCE_DATA_DIR): os.makedirs(INSTANCE_DATA_DIR) -ROOT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +ROOT_DIR = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) +) def collect_data( @@ -86,75 +89,74 @@ def create_testdata(config_file: str, force: bool) -> None: for dv_conf in workflow["dataverses"]: dv_alias = None if "create" in dv_conf: - if dv_conf["create"]: - dv = Dataverse() - dv_filename = os.path.join(ROOT_DIR, dv_conf["filename"]) - dv.from_json(read_file(dv_filename)) - if "update" in dv_conf: - for key, val in dv_conf["update"].items(): - kwargs = {key: val} - dv.set(kwargs) - dv_alias = dv.get()["alias"] - resp = api.create_dataverse(dv_conf["parent"], dv.json()) + dv = Dataverse() + dv_filename = os.path.join(ROOT_DIR, dv_conf["create"]["metadata-filename"]) + dv.from_json(read_file(dv_filename)) + if "update" in dv_conf["create"]: + for key, val in dv_conf["create"]["update"].items(): + kwargs = {key: val} + dv.set(kwargs) + dv_alias = dv.get()["alias"] + resp = api.create_dataverse(dv_conf["create"]["parent"], dv.json()) if "publish" in dv_conf: - if dv_conf["publish"]: - if not dv_alias and "alias" in dv_conf: - dv_alias = dv_conf["alias"] - resp = api.publish_dataverse(dv_alias) + if not dv_alias and "alias" in dv_conf["publish"]: + dv_alias = dv_conf["publish"]["alias"] + resp = api.publish_dataverse(dv_alias) # Datasets for ds_conf in workflow["datasets"]: pid = None if "create" in ds_conf: - if ds_conf["create"]: - ds = Dataset() - ds_filename = os.path.join(ROOT_DIR, ds_conf["filename"]) - ds.from_json(read_file(ds_filename)) - if "update" in ds_conf: - for key, val in ds_conf["update"].items(): - kwargs = {key: val} - ds.set(kwargs) - resp = api.create_dataset(dv_alias, ds.json()) - pid = resp.json()["data"]["persistentId"] - pid_idx.append(pid) + ds = Dataset() + ds_filename = os.path.join(ROOT_DIR, ds_conf["create"]["metadata-filename"]) + ds.from_json(read_file(ds_filename)) + if "update" in ds_conf["create"]: + for key, val in ds_conf["create"]["update"].items(): + kwargs = {key: val} + ds.set(kwargs) + resp = api.create_dataset(dv_alias, ds.json()) + pid = resp.json()["data"]["persistentId"] + pid_idx.append(pid) if "publish" in ds_conf: - if ds_conf["publish"]: - if not pid: - print("ERROR: PID missing!") - sys.exit() - resp = api.publish_dataset(pid, release_type="major") + if not pid: + print("ERROR: PID missing!") + sys.exit() + resp = api.publish_dataset(pid, release_type="major") # Datafiles - for df_conf in workflow["datafiles"]: - if "create" in df_conf: - if df_conf["create"]: - metadata = read_json(df_conf["metadata-filename"]) + for dataset_id, ds_datafiles in workflow["datafiles"].items(): + if int(dataset_id) == workflow["datasets"][int(dataset_id)]["id"]: + pid = pid_idx[int(dataset_id)] + else: + print("ERROR: Dataset ID not matching.") + sys.exit() + for df_conf in ds_datafiles: + if "upload" in df_conf: + metadata = read_json(df_conf["upload"]["metadata-filename"]) df = Datafile() df.set(metadata) - if "update" in df_conf: - for key, val in df_conf["update"].items(): + if "update" in df_conf["upload"]: + for key, val in df_conf["upload"]["update"].items(): kwargs = {key: val} df.set(kwargs) - pid = pid_idx[df_conf["parent"]] df.set({"pid": pid}) - filename = df_conf["filename"] + filename = df_conf["upload"]["filename"] resp = api.upload_datafile(pid, filename, df.json()) if filename[-4:] == ".sav" or filename[-4:] == ".dta": sleep(30) else: sleep(3) - if "publish-dataset" in df_conf: - if df_conf["publish-dataset"]: - resp = api.publish_dataset(pid, release_type="major") + if "publish-dataset" in df_conf: + if df_conf["publish-dataset"]: + resp = api.publish_dataset(pid, release_type="major") def remove_testdata( - config_file: str = None, - parent: str = None, + parent: str, + parent_data_type: str = "dataverse", data_types: List[str] = ["dataverses", "datasets"], - ds_published: bool = False, force: bool = False, ) -> None: if config.PRODUCTION and not force: @@ -162,20 +164,13 @@ def remove_testdata( "Delete testdata on a PRODUCTION instance not allowed. Use --force to force it." ) sys.exit() - workflow = read_json(os.path.join(ROOT_DIR, config_file)) - if "parent" in workflow: - parent = workflow["parent"] - if "data-types" in workflow: - data_types = workflow["data-types"] - if "datasets-published" in workflow: - datasets_published = workflow["datasets-published"] api = NativeApi(config.BASE_URL, config.API_TOKEN) # Clean up data = api.get_children(parent, children_types=data_types) dataverses, datasets, datafiles = dataverse_tree_walker(data) - if "parent-data-type" in workflow: + if parent_data_type == "dataverse": dataverses.append({"dataverse_alias": parent}) for ds in datasets: diff --git a/utils/config.py b/src/dvtests/utils/config.py similarity index 100% rename from utils/config.py rename to src/dvtests/utils/config.py diff --git a/src/dvtests/utils/configs/create_testdata_01.json b/src/dvtests/utils/configs/create_testdata_01.json new file mode 100644 index 0000000..09a5b7e --- /dev/null +++ b/src/dvtests/utils/configs/create_testdata_01.json @@ -0,0 +1,86 @@ +{ + "dataverses": [ + { + "create": { + "metadata-filename": "dataverse_testdata/metadata/json/dataverse/dataverse_upload_full_01.json", + "parent": ":root", + "update": { + "alias": "test_create_testdata", + "name": "Test Create Testdata" + } + }, + "publish": {} + }, + { + "create": { + "metadata-filename": "dataverse_testdata/metadata/json/dataverse/dataverse_upload_full_01.json", + "parent": "test_create_testdata" + }, + "publish": {} + } + ], + "datasets": [ + { + "id": 0, + "create": { + "metadata-filename": "dataverse_testdata/metadata/json/dataset/dataset_upload_default_full_01.json", + "parent": "test_create_testdata" + }, + "publish": {} + }, + { + "id": 1, + "create": { + "metadata-filename": "dataverse_testdata/metadata/json/dataset/dataset_upload_default_min_01.json", + "parent": "DataverseName1" + }, + "publish": {} + } + ], + "datafiles": { + "0": [ + { + "upload": { + "metadata-filename": "dataverse_testdata/metadata/json/datafile/datafile_upload_full_01.json", + "filename": "dataverse_testdata/files/jpeg/10000_image.jpeg", + "update": { + "filename": "10000_image.jpeg" + } + }, + "publish-dataset": {} + }, + { + "upload": { + "metadata-filename": "dataverse_testdata/metadata/json/datafile/datafile_upload_min_01.json", + "filename": "dataverse_testdata/files/png/10000_image.png", + "update": { + "filename": "10000_image.png" + } + }, + "publish-dataset": {} + } + ], + "1": [ + { + "upload": { + "metadata-filename": "dataverse_testdata/metadata/json/datafile/datafile_upload_min_01.json", + "filename": "dataverse_testdata/files/do/10002_data_checks.do", + "update": { + "filename": "10002_data_checks.do" + } + }, + "publish-dataset": {} + }, + { + "upload": { + "metadata-filename": "dataverse_testdata/metadata/json/datafile/datafile_upload_full_01.json", + "filename": "dataverse_testdata/files/dta/10002_da_de_v0_9.dta", + "update": { + "filename": "10002_da_de_v0_9.dta" + } + }, + "publish-dataset": {} + } + ] + } +} diff --git a/src/dvtests/utils/configs/create_testdata_02.json b/src/dvtests/utils/configs/create_testdata_02.json new file mode 100644 index 0000000..0742b4b --- /dev/null +++ b/src/dvtests/utils/configs/create_testdata_02.json @@ -0,0 +1,39 @@ +{ + "dataverses": [ + { + "create": { + "metadata-filename": "dataverse_testdata/metadata/json/dataverse/dataverse_upload_full_01.json", + "parent": ":root", + "update": { + "alias": "test_create_testdata", + "name": "Test Create Testdata" + } + }, + "publish": {} + } + ], + "datasets": [ + { + "id": 0, + "create": { + "metadata-filename": "dataverse_testdata/metadata/json/dataset/dataset_upload_default_full_01.json", + "parent": "test_create_testdata" + }, + "publish": {} + } + ], + "datafiles": { + "0": [ + { + "upload": { + "metadata-filename": "dataverse_testdata/metadata/json/datafile/datafile_upload_full_01.json", + "filename": "dataverse_testdata/files/jpeg/10000_image.jpeg", + "update": { + "filename": "10000_image.jpeg" + } + }, + "publish-dataset": {} + } + ] + } +} diff --git a/utils/__init__.py b/utils/__init__.py deleted file mode 100644 index e69de29..0000000