💥(cli) rename fetch & push to read & write

Rename `fetch` & `push` to `read` & `write` in cli for consistency with the new backend syntax.
rprin · Aug 2, 2023 · 29772a9 · 29772a9
1 parent ca21270
commit 29772a9
Show file tree

Hide file tree

Showing 20 changed files with 144 additions and 121 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,7 @@ and this project adheres to
 using the cli
 - `GET /statements` now has "mine" option which matches statements that
 have an authority field matching that of the user
+- CLI: change `push` to `write` and `fetch` to `read` [BC]
 
 ## [3.9.0] - 2023-07-21
 

diff --git a/README.md b/README.md
@@ -150,6 +150,10 @@ $ ralph COMMAND --help
 > You should substitute `COMMAND` by the target command, _e.g._ `list`, to see
 > its usage.
 
+## Migrating
+
+Some major version changes require updating persistence layers. Check out the [migration guide](https://github.com/openfun/ralph/blob/master/UPGRADE.md) for more information.
+
 ## Documentation
 
 We try our best to maintain an up-to-date reference documentation for this

diff --git a/UPGRADE.md b/UPGRADE.md
@@ -0,0 +1,17 @@
+# Upgrade
+
+All instructions to upgrade this project from one release to the next will be documented in this file. Upgrades must be run sequentially, meaning you should not skip minor/major releases while upgrading (fix releases can be skipped).
+
+This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+### 3.x.x to 4.0.0
+
+#### Upgrade history syntax
+
+CLI syntax has been changed from `fetch` & `push` to `read` & `write` affecting the command history. You must replace the command history after updating:
+- locate your history file path, which is in `{ RALPH_APP_DIR }/history.json` (defaults to `.ralph/history.json`)
+- run the commands below to update history
+```
+$ sed -i 's/"fetch"/"read"/g' { my_history_file_path }
+$ sed -i 's/"push"/"write"/g' { my_history_file_path }
+```
diff --git a/docs/contribute.md b/docs/contribute.md
@@ -103,18 +103,18 @@ it's time to play with them:
 ```bash
 # Store a JSON file in the Swift backend
 $ echo '{"id": 1, "foo": "bar"}' | \
-    ./bin/ralph push -b swift -f foo.json
+    ./bin/ralph write -b swift -f foo.json
 
 # Check that we have created a new JSON file in the Swift backend
 $ bin/ralph list -b swift
 foo.json
 
-# Fetch the content of the JSON file and index it in Elasticsearch
-$ bin/ralph fetch -b swift foo.json | \
-    bin/ralph push -b es
+# Read the content of the JSON file and index it in Elasticsearch
+$ bin/ralph read -b swift foo.json | \
+    bin/ralph write -b es
 
 # Check that we have properly indexed the JSON file in Elasticsearch
-$ bin/ralph fetch -b es
+$ bin/ralph read -b es
 {"id": 1, "foo": "bar"}
 ```
 

diff --git a/docs/index.md b/docs/index.md
@@ -2,7 +2,7 @@
 
 Ralph is a toolbox for your learning analytics, it can be used as a:
 
-- **library**, to fetch learning events from various backends, (de)serialize or
+- **library**, to read learning events from various backends, (de)serialize or
   convert them from various standard formats such as
   [xAPI](https://adlnet.gov/projects/xapi/),
 - **command-line interface** (CLI), to build data pipelines the UNIX-way™️,
@@ -38,15 +38,15 @@ base commands and UNIX standard streams (`stdin`, `stdout`) to connect them in
 a pipeline. A base example pipeline may be:
 
 ```sh
-$ ralph fetch --backend swift my_archive.gzip | \
+$ ralph read --backend swift my_archive.gzip | \
     gunzip | \
-    ralph push --backend es
+    ralph write --backend es
 ```
 
 In this small pipeline, we stream `my_archive.gzip` content from a Swift
-container to the standard output (using the `fetch` command), uncompress the
+container to the standard output (using the `read` command), uncompress the
 content (using the `gunzip` command), and bulk insert logs in an ElasticSearch
-index (using the `push` command).
+index (using the `write` command).
 
 As UNIX is beautiful, Ralph offers many powerful possibilities by combining its
 commands with other standard commands or command line tools.

diff --git a/setup.cfg b/setup.cfg
@@ -121,7 +121,7 @@ universal = 1
 max-line-length = 88
 extend-ignore = E203
 exclude =
-    .conda
+    .conda,
     .git,
     .venv,
     build,

diff --git a/src/helm/ralph/values.yaml b/src/helm/ralph/values.yaml
@@ -119,10 +119,10 @@ lrs:
 #     command:
 #       - ralph list --backend ldp --new |
 #         xargs -I {} -n 1 bash -c "
-#         ralph fetch --backend ldp {} |
+#         ralph read --backend ldp {} |
 #         gunzip |
 #         ralph extract -p gelf |
-#         ralph push \
+#         ralph write \
 #         --backend es \
 #         --es-client-options ca_certs=/usr/local/share/ca-certificates/es-cluster.pem"
 cronjobs: []

diff --git a/src/ralph/api/routers/statements.py b/src/ralph/api/routers/statements.py
@@ -229,7 +229,7 @@ async def get(
         ),
     ),
 ):
-    """Fetches a single xAPI Statement or multiple xAPI Statements.
+    """Read a single xAPI Statement or multiple xAPI Statements.
 
     LRS Specification:
     https://github.com/adlnet/xAPI-Spec/blob/1.0.3/xAPI-Communication.md#213-get-statements

diff --git a/src/ralph/backends/storage/fs.py b/src/ralph/backends/storage/fs.py
@@ -62,7 +62,7 @@ def list(self, details=False, new=False):
         logger.debug("Found %d archives", len(archives))
 
         if new:
-            archives = set(archives) - set(self.get_command_history(self.name, "fetch"))
+            archives = set(archives) - set(self.get_command_history(self.name, "read"))
             logger.debug("New archives: %d", len(archives))
 
         for archive in archives:
@@ -86,7 +86,7 @@ def read(self, name, chunk_size: int = 4096):
         self.append_to_history(
             {
                 "backend": self.name,
-                "command": "fetch",
+                "command": "read",
                 "id": name,
                 "filename": details.get("filename"),
                 "size": details.get("size"),
@@ -114,7 +114,7 @@ def write(self, stream, name, overwrite=False):
         self.append_to_history(
             {
                 "backend": self.name,
-                "command": "push",
+                "command": "write",
                 "id": name,
                 "filename": details.get("filename"),
                 "size": details.get("size"),

diff --git a/src/ralph/backends/storage/ldp.py b/src/ralph/backends/storage/ldp.py
@@ -104,7 +104,7 @@ def list(self, details=False, new=False):
         logger.debug("Found %d archives", len(archives))
 
         if new:
-            archives = set(archives) - set(self.get_command_history(self.name, "fetch"))
+            archives = set(archives) - set(self.get_command_history(self.name, "read"))
             logger.debug("New archives: %d", len(archives))
 
         for archive in archives:
@@ -130,7 +130,7 @@ def read(self, name, chunk_size=4096):
         self.append_to_history(
             {
                 "backend": self.name,
-                "command": "fetch",
+                "command": "read",
                 "id": name,
                 "filename": details.get("filename"),
                 "size": details.get("size"),

diff --git a/src/ralph/backends/storage/s3.py b/src/ralph/backends/storage/s3.py
@@ -64,7 +64,7 @@ def list(self, details=False, new=False):
         """Lists archives in the storage backend."""
         archives_to_skip = set()
         if new:
-            archives_to_skip = set(self.get_command_history(self.name, "fetch"))
+            archives_to_skip = set(self.get_command_history(self.name, "read"))
 
         try:
             paginator = self.client.get_paginator("list_objects_v2")
@@ -114,7 +114,7 @@ def read(self, name, chunk_size: int = 4096):
         self.append_to_history(
             {
                 "backend": self.name,
-                "command": "fetch",
+                "command": "read",
                 "id": name,
                 "size": size,
                 "fetched_at": now(),
@@ -141,7 +141,7 @@ def write(self, stream, name, overwrite=False):
         self.append_to_history(
             {
                 "backend": self.name,
-                "command": "push",
+                "command": "write",
                 "id": name,
                 "pushed_at": now(),
             }

diff --git a/src/ralph/backends/storage/swift.py b/src/ralph/backends/storage/swift.py
@@ -81,7 +81,7 @@ def list(self, details=False, new=False):
         """Lists files in the storage backend."""
         archives_to_skip = set()
         if new:
-            archives_to_skip = set(self.get_command_history(self.name, "fetch"))
+            archives_to_skip = set(self.get_command_history(self.name, "read"))
         with SwiftService(self.options) as swift:
             for page in swift.list(self.container):
                 if not page["success"]:
@@ -126,7 +126,7 @@ def read(self, name, chunk_size=None):
         self.append_to_history(
             {
                 "backend": self.name,
-                "command": "fetch",
+                "command": "read",
                 "id": name,
                 "size": size,
                 "fetched_at": now(),
@@ -153,7 +153,7 @@ def write(self, stream, name, overwrite=False):
         self.append_to_history(
             {
                 "backend": self.name,
-                "command": "push",
+                "command": "write",
                 "id": name,
                 "pushed_at": now(),
             }

diff --git a/src/ralph/cli.py b/src/ralph/cli.py
@@ -304,7 +304,7 @@ def wrapper(command):
 )
 @click.option(
     "-w",
-    "--write",
+    "--write-to-disk",
     is_flag=True,
     default=False,
     help="Write new credentials to the LRS authentication file.",
@@ -314,7 +314,7 @@ def auth(
     username,
     password,
     scope,
-    write,
+    write_to_disk,
     agent_ifi_mbox,
     agent_ifi_mbox_sha1sum,
     agent_ifi_openid,
@@ -392,7 +392,7 @@ def auth(
         agent=agent,
     )
 
-    if write:
+    if write_to_disk:
         logger.info("Will append new credentials to: %s", settings.AUTH_FILE)
 
         # Force Path object instantiation so that the file creation can be
@@ -561,7 +561,7 @@ def convert(from_, to_, ignore_errors, fail_on_unknown, **conversion_set_kwargs)
     "--target",
     type=str,
     default=None,
-    help="Endpoint from which to fetch events (e.g. `/statements`)",
+    help="Endpoint from which to read events (e.g. `/statements`)",
 )
 @click.option(
     "-q",
@@ -570,8 +570,8 @@ def convert(from_, to_, ignore_errors, fail_on_unknown, **conversion_set_kwargs)
     default=None,
     help="Query object as a JSON string (database and HTTP backends ONLY)",
 )
-def fetch(backend, archive, chunk_size, target, query, **options):
-    """Fetch an archive or records from a configured backend."""
+def read(backend, archive, chunk_size, target, query, **options):
+    """Read an archive or records from a configured backend."""
     logger.info(
         (
             "Fetching data from the configured %s backend "
@@ -673,9 +673,9 @@ def fetch(backend, archive, chunk_size, target, query, **options):
     "--target",
     type=str,
     default=None,
-    help="Endpoint in which to push events (e.g. `statements`)",
+    help="Endpoint in which to write events (e.g. `statements`)",
 )
-def push(
+def write(
     backend,
     archive,
     chunk_size,
@@ -686,8 +686,9 @@ def push(
     target,
     **options,
 ):
-    """Push an archive to a configured backend."""
-    logger.info("Pushing archive %s to the configured %s backend", archive, backend)
+    """Write an archive to a configured backend."""
+    logger.info("Writing archive %s to the configured %s backend", archive, backend)
+
     logger.debug("Backend parameters: %s", options)
 
     if max_num_simultaneous == 1:

diff --git a/src/tray/vars/all/main.yml b/src/tray/vars/all/main.yml
@@ -21,10 +21,10 @@ ralph_mount_es_ca_secret: false
 #       - "-c"
 #       - ralph list --backend ldp --new |
 #           xargs -I {} -n 1 bash -c "
-#             ralph fetch --backend ldp {} |
+#             ralph read --backend ldp {} |
 #             gunzip |
 #             ralph extract -p gelf |
-#             ralph push \
+#             ralph write \
 #               --backend es \
 #               --es-client-options ca_certs=/usr/local/share/ca-certificates/es-cluster.pem"
 ralph_cronjobs: []

diff --git a/tests/backends/storage/test_ldp.py b/tests/backends/storage/test_ldp.py
@@ -229,30 +229,30 @@ def mock_list(url):
     # Apply the monkeypatch for requests.post to mock_get
     monkeypatch.setattr(storage.client, "get", mock_list)
 
-    # Create a fetch history
+    # Create a read history
     fs.create_file(
         settings.HISTORY_FILE,
         contents=json.dumps(
             [
                 {
                     "backend": "ldp",
-                    "command": "fetch",
+                    "command": "read",
                     "id": "5d5c4c93-04a4-42c5-9860-f51fa4044aa1",
                     "filename": "20201002.tgz",
                     "size": 23424233,
                     "fetched_at": "2020-10-07T16:37:25.887664+00:00",
                 },
                 {
                     "backend": "ldp",
-                    "command": "fetch",
+                    "command": "read",
                     "id": "997db3eb-b9ca-485d-810f-b530a6cef7c6",
                     "filename": "20201002.tgz",
                     "size": 23424233,
                     "fetched_at": "2020-10-07T16:40:25.887664+00:00",
                 },
                 {
                     "backend": "ldp",
-                    "command": "fetch",
+                    "command": "read",
                     "id": "08075b54-8d24-42ea-a509-9f10b0e3b416",
                     "filename": "20201002.tgz",
                     "size": 23424233,
@@ -430,7 +430,7 @@ def now(cls, **kwargs):
     assert storage.history == [
         {
             "backend": "ldp",
-            "command": "fetch",
+            "command": "read",
             "id": "5d5c4c93-04a4-42c5-9860-f51fa4044aa1",
             "filename": "2020-06-16.gz",
             "size": 67906662,

diff --git a/tests/backends/storage/test_s3.py b/tests/backends/storage/test_s3.py
@@ -99,8 +99,8 @@ def test_backends_storage_s3_list_should_yield_archive_names(
     ]
 
     history = [
-        {"id": "2022-04-29.gz", "backend": "s3", "command": "fetch"},
-        {"id": "2022-04-30.gz", "backend": "s3", "command": "fetch"},
+        {"id": "2022-04-29.gz", "backend": "s3", "command": "read"},
+        {"id": "2022-04-30.gz", "backend": "s3", "command": "read"},
     ]
 
     s3 = s3()
@@ -224,7 +224,7 @@ def test_backends_storage_s3_read_with_valid_name_should_write_to_history(
     assert s3.history == [
         {
             "backend": "s3",
-            "command": "fetch",
+            "command": "read",
             "id": "2022-09-29.gz",
             "size": len(body),
             "fetched_at": freezed_now,
@@ -300,16 +300,16 @@ def test_backends_storage_s3_write_should_write_to_history_new_or_overwritten_ar
     )
 
     history = [
-        {"id": "2022-09-29.gz", "backend": "s3", "command": "fetch"},
-        {"id": "2022-09-30.gz", "backend": "s3", "command": "fetch"},
+        {"id": "2022-09-29.gz", "backend": "s3", "command": "read"},
+        {"id": "2022-09-30.gz", "backend": "s3", "command": "read"},
     ]
 
     freezed_now = datetime.datetime.now(tz=datetime.timezone.utc).isoformat()
     archive_name = "not_in_history.gz" if new_archive else "2022-09-29.gz"
     new_history_entry = [
         {
             "backend": "s3",
-            "command": "push",
+            "command": "write",
             "id": archive_name,
             "pushed_at": freezed_now,
         }
@@ -359,8 +359,8 @@ def test_backends_storage_s3_write_should_log_the_error(
     )
 
     history = [
-        {"id": "2022-09-29.gz", "backend": "s3", "command": "fetch"},
-        {"id": "2022-09-30.gz", "backend": "s3", "command": "fetch"},
+        {"id": "2022-09-29.gz", "backend": "s3", "command": "read"},
+        {"id": "2022-09-30.gz", "backend": "s3", "command": "read"},
     ]
 
     fs.create_file(settings.HISTORY_FILE, contents=json.dumps(history))
-Original file line number
+Diff line change
@@ Expand Up / @@ -121,7 +121,7 @@ universal = 1 @@
     max-line-length = 88
     extend-ignore = E203
     exclude =
-        .conda
+        .conda,
         .git,
         .venv,
         build,
@@ Expand Down @@