feat: pin interop tests. \nThis is a rough draft of a suite of tests …

…that verify that the go and js ipfs implementations behave the same when adding/removing pins to the datastore. I've found that they produce essentially the same pinsets with the exception of some ordering. There are a few tangential differences that I've noticed such as fresh go repos include the readme files as pinned while the js impl does not. Draft ipfs#2 coming up! Lots of code dedupe
JonKrone · Mar 2, 2018 · 56e767d · 56e767d
1 parent 10bb4cd
commit 56e767d
Show file tree

Hide file tree

Showing 4 changed files with 383 additions and 0 deletions.
diff --git a/test/fixtures/planets/Jupiter_from_Cassini.jpg b/test/fixtures/planets/Jupiter_from_Cassini.jpg
diff --git a/test/fixtures/planets/neptune/blue.md b/test/fixtures/planets/neptune/blue.md
@@ -0,0 +1 @@
+color: #3a55dd
diff --git a/test/fixtures/planets/neptune/wiki.md b/test/fixtures/planets/neptune/wiki.md
@@ -0,0 +1,7 @@
+Neptune is the eighth and farthest known planet from the Sun in the Solar System. In the Solar System, it is the fourth-largest planet by diameter, the third-most-massive planet, and the densest giant planet. Neptune is 17 times the mass of Earth and is slightly more massive than its near-twin Uranus, which is 15 times the mass of Earth and slightly larger than Neptune.[d] Neptune orbits the Sun once every 164.8 years at an average distance of 30.1 astronomical units (4.50×109 km). It is named after the Roman god of the sea and has the astronomical symbol ♆, a stylised version of the god Neptune's trident.
+
+Neptune is not visible to the unaided eye and is the only planet in the Solar System found by mathematical prediction rather than by empirical observation. Unexpected changes in the orbit of Uranus led Alexis Bouvard to deduce that its orbit was subject to gravitational perturbation by an unknown planet. Neptune was subsequently observed with a telescope on 23 September 1846[1] by Johann Galle within a degree of the position predicted by Urbain Le Verrier. Its largest moon, Triton, was discovered shortly thereafter, though none of the planet's remaining known 13 moons were located telescopically until the 20th century. The planet's distance from Earth gives it a very small apparent size, making it challenging to study with Earth-based telescopes. Neptune was visited by Voyager 2, when it flew by the planet on 25 August 1989.[11] The advent of the Hubble Space Telescope and large ground-based telescopes with adaptive optics has recently allowed for additional detailed observations from afar.
+
+Like Jupiter and Saturn, Neptune's atmosphere is composed primarily of hydrogen and helium, along with traces of hydrocarbons and possibly nitrogen, but it contains a higher proportion of "ices" such as water, ammonia, and methane. However, its interior, like that of Uranus, is primarily composed of ices and rock,[12] which is why Uranus and Neptune are normally considered "ice giants" to emphasise this distinction.[13] Traces of methane in the outermost regions in part account for the planet's blue appearance.[14]
+
+In contrast to the hazy, relatively featureless atmosphere of Uranus, Neptune's atmosphere has active and visible weather patterns. For example, at the time of the Voyager 2 flyby in 1989, the planet's southern hemisphere had a Great Dark Spot comparable to the Great Red Spot on Jupiter. These weather patterns are driven by the strongest sustained winds of any planet in the Solar System, with recorded wind speeds as high as 2,100 kilometres per hour (580 m/s; 1,300 mph).[15] Because of its great distance from the Sun, Neptune's outer atmosphere is one of the coldest places in the Solar System, with temperatures at its cloud tops approaching 55 K (−218 °C). Temperatures at the planet's centre are approximately 5,400 K (5,100 °C).[16][17] Neptune has a faint and fragmented ring system (labelled "arcs"), which was discovered in 1982, then later confirmed by Voyager 2.[18]
diff --git a/test/pin.js b/test/pin.js
@@ -0,0 +1,375 @@
+/* eslint-env mocha */
+'use strict'
+
+const os = require('os')
+const fs = require('fs')
+const path = require('path')
+const chai = require('chai')
+const hat = require('hat')
+const dirtyChai = require('dirty-chai')
+const expect = chai.expect
+chai.use(dirtyChai)
+
+const series = require('async/series')
+const waterfall = require('async/waterfall')
+const parallel = require('async/parallel')
+
+const DaemonFactory = require('ipfsd-ctl')
+
+describe('pin', function () {
+  this.timeout(5 * 1000)
+
+  const daemons = {}
+
+  function spawnAndStart(type, repoPath, cb) {
+    setTimeout(() =>
+      DaemonFactory.create({ type: type }).spawn({
+        repoPath,
+        disposable: false,
+      }, (err, daemon) => {
+        expect(err).to.not.exist()
+        daemons[type] = daemon
+
+        if (daemon.initialized) {
+          // repo already exists, no need to init
+          daemon.start(err => cb(err, daemon))
+        } else {
+          daemon.init((err, initRes) => {
+            err && console.log('daemon.init error:', err)
+            expect(err).to.not.exist()
+            daemon.start(err => cb(err, daemon))
+          })
+        }
+      }), 1000)
+  }
+
+  before(function (done) {
+    this.timeout(50 * 1000)
+    // DaemonFactory.create({ type: 'js' }).spawn({ disposable: false, repoPath: jsRepoPath}, cb)
+    done()
+  })
+
+  afterEach(function (done) {
+    this.timeout(25 * 1000)
+    stopDaemons(daemons, done)
+  })
+
+  describe(`go and js understand each other's stored pins`, () => {
+    // js-ipfs can read pins stored by go-ipfs
+    // tests that go's pin.flush and js' pin.load are compatible
+    it.skip('go -> js', function (done) {
+      // DONE?
+      this.timeout(20 * 1000)
+      this.slow(15000)
+      const repoPath = genRepoPath()
+      const content = String(Math.random() + Date.now())
+      let contentHash
+      let goPins
+      let jsPins
+      series([
+        cb => spawnAndStart('go', repoPath, cb),
+        cb => daemons.go.api.add(Buffer.from(content), (err, hash) => {
+          contentHash = hash[0].hash
+          cb(err)
+        }),
+        cb => daemons.go.api.pin.ls((err, res) => {
+          goPins = res
+          cb(err)
+        }),
+        cb => daemons.go.stop(cb),
+        cb => spawnAndStart('js', repoPath, cb),
+        cb => daemons.js.api.pin.ls((err, res) => {
+          jsPins = res
+          cb(err)
+        }),
+      ], errs => {
+        expect(errs).to.not.exist()
+        expect(goPins.length > 0).to.eql(true)
+        expect(jsPins).to.deep.include.members(goPins)
+        expect(goPins).to.deep.include.members(jsPins)
+        // expect(goPins).to.deep.eql(jsPins) // fails due to ordering
+      })
+    })
+
+    // go-ipfs can read pins stored by js-ipfs
+    // tests that js' pin.flush and go's pin.load are compatible
+    // skipped because go can not be spawned on a js repo due to changes in DataStore [link]
+    it.skip('js -> go', function (done) {
+      // DONE
+      this.timeout(20 * 1000)
+      this.slow(15000)
+      const repoPath = genRepoPath()
+      const content = String(Math.random() + Date.now())
+      let contentHash
+      let jsPins
+      let goPins
+      series([
+        cb => spawnAndStart('js', repoPath, cb),
+        cb =>
+          daemons.js.api.add(Buffer.from(content), (err, hash) => {
+            contentHash = hash[0].hash
+            cb(err)
+          }),
+        cb =>
+          daemons.js.api.pin.ls((err, res) => {
+            jsPins = res
+            cb(err)
+          }),
+        cb => daemons.js.stop(cb),
+        cb => spawnAndStart('go', repoPath, cb),
+        cb =>
+          daemons.go.api.pin.ls((err, res) => {
+            goPins = res
+            cb(err)
+          }),
+      ], errs => {
+        expect(errs).to.not.exist()
+        expect(jsPins.length > 0).to.eql(true)
+        expect(jsPins).to.deep.eql(goPins)
+      })
+    })
+
+    // fails! js-ipfs does not add these files to the pinset
+    it.skip('a new repo has the same pins (readme docs)', function (done) {
+      // DONE
+      this.timeout(20 * 1000)
+      parallel([
+        cb =>
+          spawnAndStart('go', genRepoPath(), (err, daemon) => {
+            daemon.api.pin.ls(cb)
+          }),
+        cb =>
+          spawnAndStart('js', genRepoPath(), (err, daemon) => {
+            daemon.api.pin.ls(cb)
+          }),
+      ], (errs, [goLs, jsLs]) => {
+        expect(errs).to.not.exist()
+        expect(goLs).to.deep.eql(jsLs)
+      })
+    })
+  })
+
+  // tests that each daemon pins larger files in the same chunks
+  it.skip('create the same indirect pins', function (done) {
+    // DONE
+    this.timeout(30 * 1000)
+    this.slow(30 * 1000)
+
+    const contentPath = 'test/fixtures/planets/Jupiter_from_Cassini.jpg'
+    const content = [{
+      path: contentPath,
+      content: fs.readFileSync(contentPath),
+    }]
+
+    parallel([
+      cb =>
+        spawnAndStart('go', genRepoPath(), (err, daemon) => {
+          removeAllPins(daemon)
+            .then(() => daemon.api.add(content, { pin: false }))
+            .then(chunks => Promise.all(
+              chunks.map(chunk => daemon.api.pin.add(chunk.hash))
+            ))
+            .then(() => daemon.api.pin.ls())
+            .then(goPins => cb(null, goPins))
+        }),
+      cb =>
+        spawnAndStart('js', genRepoPath(), (err, daemon) => {
+          removeAllPins(daemon)
+            .then(() => daemon.api.add(content, { pin: false }))
+            .then(chunks => Promise.all(
+              chunks.map(chunk => daemon.api.pin.add(chunk.hash))
+            ))
+            .then(pinAddRes => daemon.api.pin.ls())
+            .then(jsPins => cb(null, jsPins))
+        }),
+    ], (errs, [goPins, jsPins]) => {
+      expect(errs).to.not.exist()
+      expect(jsPins).to.deep.include.members(goPins)
+      expect(goPins).to.deep.include.members(jsPins)
+      // expect(jsPins).to.deep.eql(goPins) // fails due to ordering
+    })
+  })
+
+  // Pinning a large file with recursive=false results in the same direct pins
+  it.skip('pin directly', function (done) {
+    // DONE
+    this.timeout(30 * 1000)
+
+    const contentPath = 'test/fixtures/planets/Jupiter_from_Cassini.jpg'
+    const content = [{
+      path: contentPath,
+      content: fs.readFileSync(contentPath),
+    }]
+
+    parallel([
+      cb =>
+        spawnAndStart('go', genRepoPath(), (err, daemon) => {
+          removeAllPins(daemon)
+            .then(() => daemon.api.add(content, { pin: false }))
+            .then(chunks => Promise.all(
+              chunks.map(
+                chunk => daemon.api.pin.add(chunk.hash, { recursive: false })
+              )
+            ))
+            .then(() => daemon.api.pin.ls())
+            .then(goPins => cb(null, goPins))
+        }),
+      cb =>
+        spawnAndStart('js', genRepoPath(), (err, daemon) => {
+          removeAllPins(daemon)
+            .then(() => daemon.api.add(content, { pin: false }))
+            .then(chunks => Promise.all(
+              chunks.map(
+                chunk => daemon.api.pin.add(chunk.hash, { recursive: false })
+              )
+            ))
+            .then(pinAddRes => daemon.api.pin.ls())
+            .then(jsPins => cb(null, jsPins))
+        }),
+    ], (errs, [goPins, jsPins]) => {
+      expect(errs).to.not.exist()
+      expect(jsPins).to.deep.include.members(goPins)
+      expect(goPins).to.deep.include.members(jsPins)
+      // expect(jsPins).to.deep.eql(goPins) // fails due to ordering
+    })
+  })
+
+  // removing root pin removes children not part of another pinset
+  it.skip('pin recursively, remove the root pin', function (done) {
+    // DONE
+    this.timeout(20 * 1000)
+    this.slow(20 * 1000)
+
+    const contentPath = 'test/fixtures/planets/Jupiter_from_Cassini.jpg'
+    const content = [{
+      path: contentPath,
+      content: fs.readFileSync(contentPath),
+    }]
+
+    parallel([
+      cb =>
+        spawnAndStart('go', genRepoPath(), (err, daemon) => {
+          removeAllPins(daemon)
+            .then(() => daemon.api.add(content))
+            .then(chunks => {
+              const testFolder = chunks.find(chunk => chunk.path === 'test').hash
+              return daemon.api.pin.rm(testFolder)
+            })
+            .then(() => daemon.api.pin.ls())
+            .then(goPins => cb(null, goPins))
+            .catch(cb)
+        }),
+      cb =>
+        spawnAndStart('js', genRepoPath(), (err, daemon) => {
+          removeAllPins(daemon)
+            .then(() => daemon.api.add(content))
+            .then(chunks => {
+              const testFolder = chunks.find(chunk => chunk.path === 'test').hash
+              return daemon.api.pin.rm(testFolder)
+            })
+            .then(() => daemon.api.pin.ls())
+            .then(jsPins => cb(null, jsPins))
+            .catch(cb)
+        }),
+    ], (errs, [goPins, jsPins]) => {
+      expect(errs).to.not.exist()
+      expect(goPins.length).to.eql(0)
+      expect(jsPins.length).to.eql(0)
+      // expect(jsPins).to.deep.eql(goPins) // fails due to ordering
+    })
+  })
+
+  // When a pin contains the root pin of another and we remove it, it is
+  // instead kept but its type is changed to 'indirect'
+  it.only('remove a child shared by multiple pins', function (done) {
+    this.timeout(20 * 1000)
+    this.slow(20 * 1000)
+
+    const contentPath = 'test/fixtures/planets/Jupiter_from_Cassini.jpg'
+    const content = [{
+      path: contentPath,
+      content: fs.readFileSync(contentPath),
+    }]
+    let planetsFolder
+
+    parallel([
+      cb =>
+        spawnAndStart('go', genRepoPath(), (err, daemon) => {
+          removeAllPins(daemon)
+            .then(() => daemon.api.add(content, { pin: false }))
+            .then(chunks => {
+              planetsFolder = planetsFolder ||
+                chunks.find(chunk => chunk.path === 'test/fixtures/planets').hash
+              return daemon.api.pin.add(chunks.map(chunk => chunk.hash))
+                .then(() => daemon.api.pin.rm(planetsFolder))
+            })
+            .then(() => daemon.api.pin.ls())
+            .then(goPins => cb(null, goPins))
+            .catch(cb)
+        }),
+      cb =>
+        spawnAndStart('js', genRepoPath(), (err, daemon) => {
+          removeAllPins(daemon)
+            .then(() => daemon.api.add(content, { pin: false }))
+            .then(chunks => {
+              planetsFolder = planetsFolder ||
+                chunks.find(chunk => chunk.path === 'test/fixtures/planets').hash
+              return daemon.api.pin.add(chunks.map(chunk => chunk.hash))
+                .then(() => daemon.api.pin.rm(planetsFolder))
+            })
+            .then(() => daemon.api.pin.ls())
+            .then(jsPins => cb(null, jsPins))
+            .catch(cb)
+        }),
+    ], (errs, [goPins, jsPins]) => {
+      expect(errs).to.not.exist()
+      expect(goPins).to.deep.include.members(jsPins)
+      expect(jsPins).to.deep.include.members(goPins)
+      expect(goPins.find(pin => planetsFolder).type).to.eql('indirect')
+      // expect(jsPins).to.deep.eql(goPins) // fails due to ordering
+      done()
+    })
+  })
+
+  it('follow ipfs-paths', () => {
+    // both follow paths
+  })
+})
+
+function genRepoPath () {
+  return path.join(os.tmpdir(), hat())
+}
+
+function stopDaemons (daemons, callback) {
+  parallel(
+    Object.values(daemons).map(daemon => cb => daemon.stop(cb)),
+    callback
+  )
+}
+
+function removeAllPins(daemon) {
+  return daemon.api.pin.ls().then(pins => {
+    const rootPins = pins.filter(
+      pin => pin.type === 'recursive' || pin.type === 'direct'
+    )
+    return Promise.all(rootPins.map(pin => daemon.api.pin.rm(pin.hash)))
+  })
+}
+
+function createFileList(dir) {
+  return fs.readdirSync(dir)
+    .reduce((files, file) => {
+      const filePath = path.join(dir, file)
+      const isDir = fs.statSync(filePath).isDirectory()
+
+      if (isDir)
+        return files.concat(createFileList(filePath))
+      else
+        return files.concat({
+          path: filePath,
+          content: fs.readFileSync(filePath),
+        })
+    }, [])
+}
+
+// const content = createFileList('test/fixtures/planets')