fix(index): segment cache items with another subbucket (#64)

This will further help prevent individual buckets from becoming so big, they start giving filesystems a hard time because of too many inodes in a dir.

Or, you know, you can only use cacache in btrfs, that's fine too? 😏 

* fix: move hash segmenting to util module

* BREAKING: add extra sub-bucket

* reduce noise: factor out readdir w/ ignored ENOENT

* bump content/index versions

* back out module.exports = { ... }

Author: chrisdickinson

Diff for: get.js

@@ -35,7 +35,7 @@ function getData (byDigest, cache, key, opts) {
     byDigest ? Promise.resolve(null) : index.find(cache, key, opts)
   ).then(entry => {
     if (!entry && !byDigest) {
-      throw index.notFoundError(cache, key)
+      throw new index.NotFoundError(cache, key)
     }
     return read(cache, byDigest ? key : entry.digest, {
       hashAlgorithm: byDigest ? opts.hashAlgorithm : entry.hashAlgorithm,
@@ -73,7 +73,7 @@ function getStream (cache, key, opts) {
   index.find(cache, key).then(entry => {
     if (!entry) {
       return stream.emit(
-        'error', index.notFoundError(cache, key)
+        'error', new index.NotFoundError(cache, key)
       )
     }
     let memoStream

Diff for: lib/content/path.js

@@ -1,6 +1,7 @@
 'use strict'
 
 var contentVer = require('../../package.json')['cache-version'].content
+var hashToSegments = require('../util/hash-to-segments')
 var path = require('path')
 
 // Current format of content file path:
@@ -11,11 +12,9 @@ module.exports = contentPath
 function contentPath (cache, address, hashAlgorithm) {
   address = address && address.toLowerCase()
   hashAlgorithm = hashAlgorithm ? hashAlgorithm.toLowerCase() : 'sha512'
-  return path.join(
+  return path.join.apply(path, [
     cache,
     `content-v${contentVer}`,
     hashAlgorithm,
-    address.slice(0, 2),
-    address.slice(2)
-  )
+  ].concat(hashToSegments(address)))
 }

Diff for: lib/entry-index.js

@@ -1,18 +1,30 @@
 'use strict'
 
-const asyncMap = require('slide/lib/async-map')
 const contentPath = require('./content/path')
 const crypto = require('crypto')
 const fixOwner = require('./util/fix-owner')
 const fs = require('graceful-fs')
 const path = require('path')
 const Promise = require('bluebird')
-const through = require('mississippi').through
+const ms = require('mississippi')
+const hashToSegments = require('./util/hash-to-segments')
 
 const indexV = require('../package.json')['cache-version'].index
 
 const appendFileAsync = Promise.promisify(fs.appendFile)
 const readFileAsync = Promise.promisify(fs.readFile)
+const readdirAsync = Promise.promisify(fs.readdir)
+const concat = ms.concat
+const from = ms.from
+
+module.exports.NotFoundError = class NotFoundError extends Error {
+  constructor (cache, key) {
+    super('content not found')
+    this.code = 'ENOENT'
+    this.cache = cache
+    this.key = key
+  }
+}
 
 module.exports.insert = insert
 function insert (cache, key, digest, opts) {
@@ -74,71 +86,54 @@ function del (cache, key) {
 module.exports.lsStream = lsStream
 function lsStream (cache) {
   const indexDir = bucketDir(cache)
-  const stream = through.obj()
-  fs.readdir(indexDir, function (err, buckets) {
-    if (err && err.code === 'ENOENT') {
-      return stream.end()
-    } else if (err) {
-      return stream.emit('error', err)
-    } else {
-      asyncMap(buckets, (bucket, cb) => {
-        fs.readdir(path.join(indexDir, bucket), (err, files) => {
-          if (err && err.code === 'ENOENT') {
-            return cb()
-          } else if (err) {
-            return cb(err)
-          } else {
-            asyncMap(files, function (f, cb) {
-              const bpath = path.join(indexDir, bucket, f)
-              bucketEntries(cache, bpath).then(_entries => {
-                const entries = _entries.reduce((acc, entry) => {
-                  acc[entry.key] = entry
-                  return acc
-                }, {})
-                Object.keys(entries).forEach(function (k) {
-                  stream.write(formatEntry(cache, entries[k]))
-                })
-                cb()
-              }, err => {
-                if (err.code === 'ENOENT') {
-                  cb()
-                } else {
-                  cb(err)
-                }
-              })
-            }, cb)
-          }
-        })
-      }, function (err) {
-        if (err) { stream.emit('error') }
-        stream.end()
+  const stream = from.obj()
+
+  // "/cachename/*"
+  readdirOrEmpty(indexDir).map(bucket => {
+    const bucketPath = path.join(indexDir, bucket)
+
+    // "/cachename/<bucket 0xFF>/*"
+    return readdirOrEmpty(bucketPath).map(subbucket => {
+      const subbucketPath = path.join(bucketPath, subbucket)
+
+      // "/cachename/<bucket 0xFF>/<bucket 0xFF>/*"
+      return readdirOrEmpty(subbucketPath).map(entry => {
+        const getKeyToEntry = bucketEntries(
+          cache,
+          path.join(subbucketPath, entry)
+        ).reduce((acc, entry) => {
+          acc.set(entry.key, entry)
+          return acc
+        }, new Map())
+
+        return getKeyToEntry.then(reduced => {
+          return Array.from(reduced.values()).map(
+            entry => stream.push(formatEntry(cache, entry))
+          )
+        }).catch({code: 'ENOENT'}, nop)
       })
-    }
+    })
+  }).then(() => {
+    stream.push(null)
+  }, err => {
+    stream.emit('error', err)
   })
+
   return stream
 }
 
 module.exports.ls = ls
 function ls (cache) {
-  const entries = {}
   return Promise.fromNode(cb => {
-    lsStream(cache).on('finish', function () {
-      cb(null, entries)
-    }).on('data', function (d) {
-      entries[d.key] = d
-    }).on('error', cb)
+    lsStream(cache).on('error', cb).pipe(concat(entries => {
+      cb(null, entries.reduce((acc, xs) => {
+        acc[xs.key] = xs
+        return acc
+      }, {}))
+    }))
   })
 }
 
-module.exports.notFoundError = notFoundError
-function notFoundError (cache, key) {
-  const err = new Error('content not found')
-  err.code = 'ENOENT'
-  err.cache = cache
-  err.key = key
-  return err
-}
-
 function bucketEntries (cache, bucket, filter) {
   return readFileAsync(
     bucket, 'utf8'
@@ -172,7 +167,9 @@ function bucketDir (cache) {
 module.exports._bucketPath = bucketPath
 function bucketPath (cache, key) {
   const hashed = hashKey(key)
-  return path.join(bucketDir(cache), hashed.slice(0, 2), hashed.slice(2))
+  return path.join.apply(path, [bucketDir(cache)].concat(
+    hashToSegments(hashed)
+  ))
 }
 
 module.exports._hashKey = hashKey
@@ -193,3 +190,10 @@ function formatEntry (cache, entry) {
     metadata: entry.metadata
   }
 }
+
+function readdirOrEmpty (dir) {
+  return readdirAsync(dir).catch({code: 'ENOENT'}, () => [])
+}
+
+function nop () {
+}

Diff for: lib/util/hash-to-segments.js

@@ -0,0 +1,11 @@
+'use strict'
+
+module.exports = hashToSegments
+
+function hashToSegments (hash) {
+  return [
+    hash.slice(0, 2),
+    hash.slice(2, 4),
+    hash.slice(4)
+  ]
+}

Diff for: package.json

@@ -2,8 +2,8 @@
   "name": "cacache",
   "version": "6.0.1",
   "cache-version": {
-    "content": "1",
-    "index": "1"
+    "content": "2",
+    "index": "2"
   },
   "description": "General content-addressable cache system that maintains a filesystem registry of file data.",
   "main": "index.js",
@@ -58,7 +58,6 @@
     "once": "^1.4.0",
     "promise-inflight": "^1.0.1",
     "rimraf": "^2.6.1",
-    "slide": "^1.1.6",
     "unique-filename": "^1.1.0"
   },
   "devDependencies": {