feat: support any IPLD decoder (nftstorage#587)

opendidx · Oct 20, 2021 · 278c9e2 · 278c9e2
1 parent 7e31fbf
commit 278c9e2
Show file tree

Hide file tree

Showing 5 changed files with 91 additions and 14 deletions.
diff --git a/packages/client/package.json b/packages/client/package.json
@@ -44,12 +44,14 @@
     "@web-std/file": "^1.1.0",
     "@web-std/form-data": "^2.1.0",
     "carbites": "^1.0.6",
+    "multiformats": "^9.4.8",
     "p-retry": "^4.6.1",
     "streaming-iterables": "^6.0.0"
   },
   "devDependencies": {
     "@ipld/car": "^3.1.16",
     "@ipld/dag-cbor": "^6.0.9",
+    "@ipld/dag-json": "^8.0.1",
     "@ssttevee/multipart-parser": "0.1.9",
     "@types/mocha": "^9.0.0",
     "hundreds": "0.0.9",
@@ -62,7 +64,6 @@
     "just-safe-set": "^2.2.1",
     "mocha": "^9.1.0",
     "multicodec": "3.0.1",
-    "multiformats": "^9.4.3",
     "multihashing-async": "^2.1.2",
     "npm-run-all": "^4.1.5",
     "nyc": "15.1.0",

diff --git a/packages/client/src/lib.js b/packages/client/src/lib.js
@@ -26,6 +26,8 @@ const MAX_STORE_RETRIES = 5
 const MAX_CONCURRENT_UPLOADS = 3
 const MAX_CHUNK_SIZE = 1024 * 1024 * 10 // chunk to ~10MB CARs
 
+/** @typedef {import('multiformats/block').BlockDecoder<any, any>} AnyBlockDecoder */
+
 /**
  * @implements API.Service
  */
@@ -103,15 +105,18 @@ class NFTStorage {
   /**
    * @param {API.Service} service
    * @param {Blob|API.CarReader} car
-   * @param {{onStoredChunk?: (size: number) => void}} [options]
+   * @param {{
+   *   onStoredChunk?: (size: number) => void
+   *   decoders?: AnyBlockDecoder[]
+   * }} [options]
    * @returns {Promise<API.CIDString>}
    */
-  static async storeCar({ endpoint, token }, car, { onStoredChunk } = {}) {
+  static async storeCar({ endpoint, token }, car, { onStoredChunk, decoders } = {}) {
     const targetSize = MAX_CHUNK_SIZE
     const splitter =
       car instanceof Blob
-        ? await TreewalkCarSplitter.fromBlob(car, targetSize)
-        : new TreewalkCarSplitter(car, targetSize)
+        ? await TreewalkCarSplitter.fromBlob(car, targetSize, { decoders })
+        : new TreewalkCarSplitter(car, targetSize, { decoders })
 
     const upload = transform(
       MAX_CONCURRENT_UPLOADS,
@@ -321,7 +326,14 @@ class NFTStorage {
    * console.assert(cid === expectedCid)
    * ```
    * @param {Blob|API.CarReader} car
-   * @param {{onStoredChunk?: (size: number) => void}} [options]
+   * @param {object} [options]
+   * @param {(size: number) => void} [options.onStoredChunk] Callback called
+   * after each chunk of data has been uploaded. By default, data is split into
+   * chunks of around 10MB. It is passed the actual chunk size in bytes.
+   * @param {AnyBlockDecoder[]} [options.decoders] Additional IPLD block
+   * decoders. Used to interpret the data in the CAR file and split it into
+   * multiple chunks. Note these are only required if the CAR file was not
+   * encoded using the default encoders: `dag-pb`, `dag-cbor` and `raw`.
    */
   storeCar(car, options) {
     return NFTStorage.storeCar(this, car, options)

diff --git a/packages/client/src/lib/interface.ts b/packages/client/src/lib/interface.ts
@@ -1,8 +1,10 @@
 import type { CID } from 'multiformats'
 export type { CID }
 
-import type { RootsReader, BlockReader } from '@ipld/car/api'
-type CarReader = RootsReader & BlockReader
+import type { BlockDecoder } from 'multiformats/block'
+export type { BlockDecoder }
+
+import type { CarReader } from '@ipld/car/api'
 export type { CarReader }
 
 /**
@@ -55,7 +57,21 @@ export interface API {
   storeCar(
     service: Service,
     content: Blob | CarReader,
-    options?: { onStoredChunk?: (size: number) => void }
+    options?: {
+      /**
+       * Callback called after each chunk of data has been uploaded. By default,
+       * data is split into chunks of around 10MB. It is passed the actual chunk
+       * size in bytes.
+       */
+      onStoredChunk?: (size: number) => void
+      /**
+       * Additional IPLD block decoders. Used to interpret the data in the CAR
+       * file and split it into multiple chunks. Note these are only required if
+       * the CAR file was not encoded using the default encoders: `dag-pb`,
+       * `dag-cbor` and `raw`.
+       */
+      decoders?: BlockDecoder<any, any>[]
+    }
   ): Promise<CIDString>
   /**
    * Stores a directory of files and returns a CID. Provided files **MUST**

diff --git a/packages/client/test/lib.spec.js b/packages/client/test/lib.spec.js
@@ -5,7 +5,9 @@ import { CID } from 'multiformats'
 import { pack } from 'ipfs-car/pack'
 import { CarWriter } from '@ipld/car'
 import * as dagCbor from '@ipld/dag-cbor'
+import * as dagJson from '@ipld/dag-json'
 import { garbage } from 'ipld-garbage'
+import { encode } from 'multiformats/block'
 import { sha256 } from 'multiformats/hashes/sha2'
 
 const DWEB_LINK = 'dweb.link'
@@ -154,6 +156,17 @@ describe('client', () => {
       assert.ok(uploadedChunks >= 12)
       assert.equal(cid, expectedCid)
     })
+
+    it('upload CAR with non-default decoder', async () => {
+      const client = new NFTStorage({ token, endpoint })
+      const block = await encode({ value: { hello: 'world' }, codec: dagJson, hasher: sha256 })
+      const { writer, out } = CarWriter.create([block.cid])
+      writer.put(block)
+      writer.close()
+      const reader = await CarReader.fromIterable(out)
+      const cid = await client.storeCar(reader, { decoders: [dagJson] })
+      assert.equal(cid, block.cid.toString(), 'returned cid matches the CAR')
+    })
   })
 
   describe('upload dir', () => {

diff --git a/yarn.lock b/yarn.lock
@@ -398,6 +398,14 @@
     cborg "^1.2.1"
     multiformats "^9.0.0"
 
+"@ipld/dag-json@^8.0.1":
+  version "8.0.1"
+  resolved "https://registry.yarnpkg.com/@ipld/dag-json/-/dag-json-8.0.1.tgz#63d36495396e3d7dfc7d12f09d19bb92096ef6cc"
+  integrity sha512-+vl+mCeHWOih269/+E2oQ3vWnDSt9K5PKvnwrmwzgFJw3u/UHwbi4RwdYpvLupvDIkKNnGGURQYdxh35DlHsSw==
+  dependencies:
+    cborg "^1.4.0"
+    multiformats "^9.0.0"
+
 "@ipld/dag-pb@^2.0.2", "@ipld/dag-pb@^2.1.1":
   version "2.1.11"
   resolved "https://registry.yarnpkg.com/@ipld/dag-pb/-/dag-pb-2.1.11.tgz#5973983fe3c8ffa26ad1aff932fdb9d469bffbc4"
@@ -2394,7 +2402,7 @@ catering@^2.0.0:
   dependencies:
     queue-tick "^1.0.0"
 
-cborg@^1.0.4, cborg@^1.2.1:
+cborg@^1.0.4, cborg@^1.2.1, cborg@^1.4.0:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/cborg/-/cborg-1.5.1.tgz#0407f240c7c19632f0e571fed7270ee472f050af"
   integrity sha512-GKCylZR7os3Q9X+U3DiARfeFKQUdcZMAP8EKFSE91YbhJsxV71Z6PMOT2osVWprb+iWf6viyqD7peEkK0QCAAw==
@@ -3530,11 +3538,21 @@ es6-weak-map@^2.0.3:
     es6-iterator "^2.0.3"
     es6-symbol "^3.1.1"
 
-esbuild@0.12.21, esbuild@0.12.26, esbuild@^0.12.15, esbuild@^0.8.17:
+esbuild@0.12.26:
+  version "0.12.26"
+  resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.12.26.tgz#35f2d58ac3fa4629df24aa4d6fd72feb5522e94b"
+  integrity sha512-YmTkhPKjvTJ+G5e96NyhGf69bP+hzO0DscqaVJTi5GM34uaD4Ecj7omu5lJO+NrxCUBRhy2chONLK1h/2LwoXA==
+
+esbuild@^0.12.15:
   version "0.12.21"
   resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.12.21.tgz#7ff32a9ac73ce4310f9cb61ea4c3da9756570d46"
   integrity sha512-7hyXbU3g94aREufI/5nls7Xcc+RGQeZWZApm6hoBaFvt2BPtpT4TjFMQ9Tb1jU8XyBGz00ShmiyflCogphMHFQ==
 
+esbuild@^0.8.17:
+  version "0.8.57"
+  resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.8.57.tgz#a42d02bc2b57c70bcd0ef897fe244766bb6dd926"
+  integrity sha512-j02SFrUwFTRUqiY0Kjplwjm1psuzO1d6AjaXKuOR9hrY0HuPsT6sV42B6myW34h1q4CRy+Y3g4RU/cGJeI/nNA==
+
 escalade@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
@@ -4659,11 +4677,18 @@ graphql-typed-client@1.7.4:
     graphql-js-tree "0.0.1"
     yargs "^16.1.1"
 
-graphql@*, graphql@^14.5.8, graphql@^15.4.0, graphql@^15.5.1:
+graphql@*, graphql@^15.4.0:
   version "15.6.1"
   resolved "https://registry.yarnpkg.com/graphql/-/graphql-15.6.1.tgz#9125bdf057553525da251e19e96dab3d3855ddfc"
   integrity sha512-3i5lu0z6dRvJ48QP9kFxBkJ7h4Kso7PS8eahyTFz5Jm6CvQfLtNIE8LX9N6JLnXTuwR+sIYnXzaWp6anOg0QQw==
 
+graphql@^14.5.8:
+  version "14.7.0"
+  resolved "https://registry.yarnpkg.com/graphql/-/graphql-14.7.0.tgz#7fa79a80a69be4a31c27dda824dc04dac2035a72"
+  integrity sha512-l0xWZpoPKpppFzMfvVyFmp9vLN7w/ZZJPefUicMCepfJeQ8sMcztloGYY9DfjVPo6tIUDzU5Hw3MUbIjj9AVVA==
+  dependencies:
+    iterall "^1.2.2"
+
 growl@1.10.5:
   version "1.10.5"
   resolved "https://registry.yarnpkg.com/growl/-/growl-1.10.5.tgz#f2735dc2283674fa67478b10181059355c369e5e"
@@ -6236,7 +6261,7 @@ it-to-stream@^1.0.0:
     p-fifo "^1.0.0"
     readable-stream "^3.6.0"
 
-iterall@^1.1.3, iterall@^1.2.1:
+iterall@^1.1.3, iterall@^1.2.1, iterall@^1.2.2:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.3.0.tgz#afcb08492e2915cbd8a0884eb93a8c94d0d72fea"
   integrity sha512-QZ9qOMdF+QLHxy1QIpUHUU1D5pS2CG2P69LF6L6CPjPYA/XMOmKV3PZpawHoAjHNyB0swdVTRxdYT4tbBbxqwg==
@@ -8354,11 +8379,21 @@ premove@^3.0.1:
   resolved "https://registry.yarnpkg.com/premove/-/premove-3.0.1.tgz#3e6f1508e963716f317fe0d39785758485d02cb4"
   integrity sha512-lH6+7lVMTu7Fl3NdfptcY7E5ElRVhAOpW0mfJq8qPex84KEaT7TOXYvGLW2PFp+wpWrvNMAn3RNYyy+QKx2kiA==
 
-prettier@2.4.0, prettier@^1.19.1, prettier@^2.4.1:
+prettier@2.4.0:
   version "2.4.0"
   resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.4.0.tgz#85bdfe0f70c3e777cf13a4ffff39713ca6f64cba"
   integrity sha512-DsEPLY1dE5HF3BxCRBmD4uYZ+5DCbvatnolqTqcxEgKVZnL2kUfyu7b8pPQ5+hTBkdhU9SLUmK0/pHb07RE4WQ==
 
+prettier@^1.19.1:
+  version "1.19.1"
+  resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.19.1.tgz#f7d7f5ff8a9cd872a7be4ca142095956a60797cb"
+  integrity sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==
+
+prettier@^2.4.1:
+  version "2.4.1"
+  resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.4.1.tgz#671e11c89c14a4cfc876ce564106c4a6726c9f5c"
+  integrity sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==
+
 prismjs@^1.22.0, prismjs@~1.25.0:
   version "1.25.0"
   resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.25.0.tgz#6f822df1bdad965734b310b315a23315cf999756"