Add --force option to allow overwrite local contents on download

NatLibFi · juhoinkinen · Apr 23, 2024 · Feb 1, 2024 · Feb 5, 2024 · Feb 6, 2024
commit 7666de8a9a0dcb7d575c0ae7ae285f0d94244648
diff --git a/annif/cli.py b/annif/cli.py
@@ -6,7 +6,6 @@
 import json
 import os.path
 import re
-import shutil
 import sys
 from fnmatch import fnmatch
 
@@ -606,34 +605,34 @@
    with the projects configuration to the specified Hugging Face Hub repository.
    An authentication token and commit message can be given with options.
    """
    projects = [
        proj
        for proj in annif.registry.get_projects(min_access=Access.private).values()
        if fnmatch(proj.project_id, project_ids_pattern)
    ]
    click.echo(f"Uploading project(s): {', '.join([p.project_id for p in projects])}")

    commit_message = (
        commit_message
        if commit_message is not None
        else f"Upload project(s) {project_ids_pattern} with Annif"
    )

    project_dirs = {p.datadir for p in projects}
    vocab_dirs = {p.vocab.datadir for p in projects}
    data_dirs = project_dirs.union(vocab_dirs)

    for data_dir in data_dirs:
        zip_path = data_dir.split(os.path.sep, 1)[1] + ".zip"  # TODO Check this
        fobj = cli_util.archive_dir(data_dir)
        cli_util.upload_to_hf_hub(fobj, zip_path, repo_id, token, commit_message)
        fobj.close()

    for project in projects:
        config_path = project.project_id + ".cfg"
        fobj = cli_util.write_config(project)
        cli_util.upload_to_hf_hub(fobj, config_path, repo_id, token, commit_message)
        fobj.close()


 @cli.command("download")
@@ -651,8 +650,15 @@
     hash.
     """,
 )
+@click.option(
+    "--force",
+    "-f",
+    default=False,
+    is_flag=True,
+    help="Replace an existing project/vocabulary/config with the downloaded one",
+)
 @cli_util.common_options
-def run_download(project_ids_pattern, repo_id, token, revision):
+def run_download(project_ids_pattern, repo_id, token, revision, force):
     """
     Download selected projects and their vocabularies from a Hugging Face Hub repository
     \f
@@ -664,30 +670,30 @@
    be given with options.
    """

    project_ids = cli_util.get_selected_project_ids_from_hf_hub(
        project_ids_pattern, repo_id, token, revision
    )
    click.echo(f"Downloading project(s): {', '.join(project_ids)}")

    if not os.path.isdir("projects.d"):
        os.mkdir("projects.d")
    vocab_ids = set()
    for project_id in project_ids:
         project_zip_local_cache_path = cli_util.download_from_hf_hub(
             f"projects/{project_id}.zip", repo_id, token, revision
         )
-        cli_util.unzip(project_zip_local_cache_path)
+        cli_util.unzip(project_zip_local_cache_path, force)
         local_config_cache_path = cli_util.download_from_hf_hub(
             f"{project_id}.cfg", repo_id, token, revision
         )
         vocab_ids.add(cli_util.get_vocab_id(local_config_cache_path))
-        shutil.copy(local_config_cache_path, "projects.d")  # TODO Disallow overwrite
+        cli_util.move_project_config(local_config_cache_path, force)
 
     for vocab_id in vocab_ids:
         vocab_zip_local_cache_path = cli_util.download_from_hf_hub(
             f"vocabs/{vocab_id}.zip", repo_id, token, revision
         )
-        cli_util.unzip(vocab_zip_local_cache_path)
+        cli_util.unzip(vocab_zip_local_cache_path, force)
 
 
 @cli.command("completion")

diff --git a/annif/cli_util.py b/annif/cli_util.py
@@ -2,12 +2,14 @@
 
 from __future__ import annotations
 
+import binascii
 import collections
 import configparser
 import io
 import itertools
 import os
 import pathlib
+import shutil
 import sys
 import tempfile
 import zipfile
@@ -238,55 +240,55 @@


 def _is_train_file(fname):
    train_file_patterns = ("-train", "tmp-")
    for pat in train_file_patterns:
        if pat in fname:
            return True
    return False


 def archive_dir(data_dir):
    fp = tempfile.TemporaryFile()
    path = pathlib.Path(data_dir)
    fpaths = [fpath for fpath in path.glob("**/*") if not _is_train_file(fpath.name)]
    with zipfile.ZipFile(fp, mode="w") as zfile:
        for fpath in fpaths:
            logger.debug(f"Adding {fpath}")
            zfile.write(fpath)
    fp.seek(0)
    return fp


 def write_config(project):
    fp = tempfile.TemporaryFile(mode="w+t")
    config = configparser.ConfigParser()
    config[project.project_id] = project.config
    config.write(fp)  # This needs tempfile in text mode
    fp.seek(0)
    # But for upload fobj needs to be in binary mode
    return io.BytesIO(fp.read().encode("utf8"))


 def upload_to_hf_hub(fileobj, filename, repo_id, token, commit_message):
    from huggingface_hub import HfApi
    from huggingface_hub.utils import HfHubHTTPError, HFValidationError

    api = HfApi()
    try:
        api.upload_file(
            path_or_fileobj=fileobj,
            path_in_repo=filename,
            repo_id=repo_id,
            token=token,
            commit_message=commit_message,
        )
    except (HfHubHTTPError, HFValidationError) as err:
        raise OperationFailedException(str(err))


 def get_selected_project_ids_from_hf_hub(project_ids_pattern, repo_id, token, revision):
    all_repo_file_paths = _list_files_in_hf_hub(repo_id, token, revision)
    return [
        path.rsplit(".zip")[0].split("projects/")[1]  # TODO Try-catch this
        for path in all_repo_file_paths
        if fnmatch(path, f"projects/{project_ids_pattern}.zip")
@@ -294,39 +296,81 @@


 def _list_files_in_hf_hub(repo_id, token, revision):
    from huggingface_hub import list_repo_files

    return [
        repofile
        for repofile in list_repo_files(repo_id=repo_id, token=token, revision=revision)
    ]


 def download_from_hf_hub(filename, repo_id, token, revision):
    from huggingface_hub import hf_hub_download
    from huggingface_hub.utils import HfHubHTTPError, HFValidationError

    try:
        return hf_hub_download(
            repo_id=repo_id,
            filename=filename,
            token=token,
            revision=revision,
        )
    except (HfHubHTTPError, HFValidationError) as err:
         raise OperationFailedException(str(err))
 
 
-def unzip(source_path):
-    with zipfile.ZipFile(source_path, "r") as zfile:
-        zfile.extractall()  # TODO Disallow overwrite
+def unzip(src_path, force):
+    with zipfile.ZipFile(src_path, "r") as zfile:
+        for member in zfile.infolist():
+            if os.path.exists(member.filename) and not force:
+                if _is_existing_identical(member):
+                    logger.debug(
+                        f"Skipping unzip of {member.filename}; already in place"
+                    )
+                else:
+                    click.echo(
+                        f"Not overwriting {member.filename} (use --force to override)"
+                    )
+            else:
+                logger.debug(f"Unzipping {member.filename}")
+                zfile.extract(member)
+
+
+def move_project_config(src_path, force):
+    dst_path = os.path.join("projects.d", os.path.basename(src_path))
+    if os.path.exists(dst_path) and not force:
+        if _compute_crc32(dst_path) == _compute_crc32(src_path):
+            logger.debug(
+                f"Skipping move of {os.path.basename(src_path)}; already in place"
+            )
+        else:
+            click.echo(f"Not overwriting {dst_path} (use --force to override)")
+    else:
+        shutil.copy(src_path, dst_path)
+
+
+def _is_existing_identical(member):
+    file_crc = _compute_crc32(member.filename)
+    return file_crc == member.CRC
+
+
+def _compute_crc32(path):
+    if os.path.isdir(path):
+        return 0
+
+    size = 1024 * 1024 * 10  # 10 MiB chunks
+    with open(path, "rb") as fp:
+        crcval = 0
+        while chunk := fp.read(size):
+            crcval = binascii.crc32(chunk, crcval)
+    return crcval
 
 
 def get_vocab_id(config_path):
    config = configparser.ConfigParser()
    config.read(config_path)
    section = config.sections()[0]
    return config[section]["vocab"]


 def _get_completion_choices(