Render npm package readmes

3 files changed, 104 insertions, 7 deletions

diff --git a/lib/app.js b/lib/app.js
index 65d404a..1705d1c 100644
--- a/lib/app.js
+++ b/lib/app.js
@@ -12,6 +12,8 @@ var Serve = require('./serve')
 var Render = require('./render')
 var Git = require('./git')
 var cat = require('pull-cat')
+var proc = require('child_process')
+var toPull = require('stream-to-pull-stream')
 
 module.exports = App
 
@@ -40,11 +42,15 @@ function App(sbot, config) {
   this.unboxContent = memo({cache: lru(100)}, sbot.private.unbox)
   this.reverseNameCache = lru(500)
   this.reverseEmojiNameCache = lru(500)
+  this.getBlobSize = memo({cache: this.blobSizeCache = lru(100)},
+    sbot.blobs.size.bind(sbot.blobs))
 
   this.unboxMsg = this.unboxMsg.bind(this)
 
   this.render = new Render(this, this.opts)
   this.git = new Git(this)
+
+  this.monitorBlobWants()
 }
 
 App.prototype.go = function () {
@@ -179,9 +185,12 @@ App.prototype.publish = function (content, cb) {
 }
 
 App.prototype.wantSizeBlob = function (id, cb) {
+  // only want() the blob if we don't already have it
+  var self = this
   var blobs = this.sbot.blobs
   blobs.size(id, function (err, size) {
     if (size != null) return cb(null, size)
+    self.blobWants[id] = true
     blobs.want(id, function (err) {
       if (err) return cb(err)
       blobs.size(id, cb)
@@ -565,3 +574,50 @@ App.prototype.blobMentions = function (opts) {
     ]
   })
 }
+
+App.prototype.monitorBlobWants = function () {
+  var self = this
+  self.blobWants = {}
+  pull(
+    this.sbot.blobs.createWants(),
+    pull.drain(function (wants) {
+      for (var id in wants) {
+        if (wants[id] < 0) self.blobWants[id] = true
+        else delete self.blobWants[id]
+        self.blobSizeCache.remove(id)
+      }
+    }, function (err) {
+      if (err) console.trace(err)
+    })
+  )
+}
+
+App.prototype.getBlobState = function (id, cb) {
+  var self = this
+  if (self.blobWants[id]) return cb(null, 'wanted')
+  self.getBlobSize(id, function (err, size) {
+    if (err) return cb(err)
+    cb(null, size != null)
+  })
+}
+
+App.prototype.getNpmReadme = function (tarballId, cb) {
+  var self = this
+  // TODO: make this portable, and handle plaintext readmes
+  var tar = proc.spawn('tar', ['--ignore-case', '-Oxz',
+    'package/README.md', 'package/readme.markdown', 'package/readme.mkd'])
+  var done = multicb({pluck: 1, spread: true})
+  pull(
+    self.sbot.blobs.get(tarballId),
+    toPull.sink(tar.stdin, done())
+  )
+  pull(
+    toPull.source(tar.stdout),
+    pull.collect(done())
+  )
+  done(function (err, _, bufs) {
+    if (err) return cb(err)
+    var text = Buffer.concat(bufs).toString('utf8')
+    cb(null, text, true)
+  })
+}
diff --git a/lib/render.js b/lib/render.js
index 6001184..5a0ddaa 100644
--- a/lib/render.js
+++ b/lib/render.js
@@ -422,12 +422,14 @@ Render.prototype.npmPackageMention = function (link, opts, cb) {
   var self = this
   var done = multicb({pluck: 1, spread: true})
   var base = '/npm/' + (opts.author ? u.escapeId(link.author) + '/' : '')
-  var pathWithAuthor = '/npm/' +
+  var pathWithAuthor = opts.withAuthor ? '/npm/' +
     u.escapeId(link.author) + '/' +
       (opts.name ? opts.name + '/' +
         (opts.version ? opts.version + '/' +
-          (opts.distTag ? opts.distTag + '/' : '') : '') : '')
-  self.app.getAbout(link.author, function (err, about) {
+          (opts.distTag ? opts.distTag + '/' : '') : '') : '') : ''
+  self.app.getAbout(link.author, done())
+  self.app.getBlobState(link.link, done())
+  done(function (err, about, blobState) {
     if (err) return cb(err)
     cb(null, h('tr', [
       opts.withAuthor ? h('td', h('a', {
@@ -452,7 +454,20 @@ Render.prototype.npmPackageMention = function (link, opts, cb) {
       h('td', typeof link.link === 'string' ? h('code', h('a', {
         href: self.toUrl('/links/' + link.link),
         title: 'package tarball'
-      }, link.link.substr(0, 8) + '…')) : '')
+      }, link.link.substr(0, 8) + '…')) : ''),
+      h('td',
+        blobState === 'wanted' ?
+        'fetching...'
+        : blobState ? h('a', {
+          href: self.toUrl('/npm-readme/' + encodeURIComponent(link.link)),
+          title: 'package contents'
+        }, 'readme')
+        : h('form', {action: '', method: 'post'},
+          h('input', {type: 'hidden', name: 'action', value: 'want-blobs'}),
+          h('input', {type: 'hidden', name: 'async_want', value: '1'}),
+          h('input', {type: 'hidden', name: 'blob_ids', value: link.link}),
+          h('input', {type: 'submit', value: 'fetch'})
+        ))
     ]))
   })
 }
diff --git a/lib/serve.js b/lib/serve.js
index 5ebfa47..c162e42 100644
--- a/lib/serve.js
+++ b/lib/serve.js
@@ -201,8 +201,9 @@ Serve.prototype.wantBlobs = function (cb) {
   if (!ids.every(u.isRef)) return cb(new Error('bad blob ids ' + ids.join(',')))
   var done = multicb({pluck: 1})
   ids.forEach(function (id) {
-    self.app.sbot.blobs.want(id, done())
+    self.app.wantSizeBlob(id, done())
   })
+  if (self.data.async_want) return cb()
   done(function (err) {
     if (err) return cb(err)
     // self.note = h('div', 'wanted blobs: ' + ids.join(', ') + '.')
@@ -303,6 +304,7 @@ Serve.prototype.path = function (url) {
     case '/git': return this.git(m[2])
     case '/image': return this.image(m[2])
     case '/npm': return this.npm(m[2])
+    case '/npm-readme': return this.npmReadme(m[2])
   }
   return this.respond(404, 'Not found')
 }
@@ -1911,7 +1913,8 @@ Serve.prototype.npm = function (url) {
           ph('td', 'version'),
           ph('td', 'tag'),
           ph('td', 'size'),
-          ph('td', 'tarball')
+          ph('td', 'tarball'),
+          ph('td', 'readme')
         ])),
         ph('tbody', pull(
           self.app.blobMentions({
@@ -1934,7 +1937,30 @@ Serve.prototype.npm = function (url) {
         ))
       ])
     ]),
-    self.wrapPage('npm:'),
+    self.wrapPage(prefix),
+    self.respondSink(200)
+  )
+}
+
+Serve.prototype.npmReadme = function (url) {
+  var self = this
+  var id = decodeURIComponent(url.substr(1))
+  return pull(
+    ph('section', {}, [
+      ph('h3', [
+        'npm readme for ',
+        ph('a', {href: '/links/' + id}, id.substr(0, 8) + '…')
+      ]),
+      ph('blockquote', u.readNext(function (cb) {
+        self.app.getNpmReadme(id, function (err, readme, isMarkdown) {
+          if (err) return cb(null, ph('div', u.renderError(err).outerHTML))
+          cb(null, isMarkdown
+            ? ph('div', self.app.render.markdown(readme))
+            : ph('pre', readme))
+        })
+      }))
+    ]),
+    self.wrapPage('npm readme'),
     self.respondSink(200)
   )
 }