diff options
| -rw-r--r-- | lib/app.js | 32 | ||||
| -rw-r--r-- | lib/git.js | 380 | ||||
| -rw-r--r-- | lib/render-msg.js | 22 | ||||
| -rw-r--r-- | lib/render.js | 8 | ||||
| -rw-r--r-- | lib/serve.js | 117 | ||||
| -rw-r--r-- | lib/util.js | 20 | ||||
| -rw-r--r-- | package.json | 2 |
7 files changed, 571 insertions, 10 deletions
@@ -11,6 +11,7 @@ var Contacts = require('ssb-contact') var About = require('./about') var Serve = require('./serve') var Render = require('./render') +var Git = require('./git') module.exports = App @@ -41,6 +42,7 @@ function App(sbot, config) { this.unboxMsg = this.unboxMsg.bind(this) this.render = new Render(this, this.opts) + this.git = new Git(this) } App.prototype.go = function () { @@ -186,6 +188,36 @@ App.prototype.pushBlob = function (id, cb) { this.sbot.blobs.push(id, cb) } +App.prototype.readBlob = function (link, opts) { + link = u.toLink(link) + opts = opts || {} + return this.sbot.blobs.get({ + hash: link.link, + size: link.size, + start: opts.start, + end: opts.end, + }) +} + +App.prototype.ensureHasBlobs = function (links, cb) { + var self = this + var done = multicb({pluck: 1}) + links.forEach(function (link) { + var cb = done() + self.sbot.blobs.size(link.link, function (err, size) { + if (err) cb(err) + else if (size == null) cb(null, link) + else cb() + }) + }) + done(function (err, missingLinks) { + if (err) console.trace(err) + missingLinks = missingLinks.filter(Boolean) + if (missingLinks.length == 0) return cb() + return cb({name: 'BlobNotFoundError', links: missingLinks}) + }) +} + App.prototype.getReverseNameSync = function (name) { var id = this.reverseNameCache.get(name) return id diff --git a/lib/git.js b/lib/git.js new file mode 100644 index 0000000..36e4587 --- /dev/null +++ b/lib/git.js @@ -0,0 +1,380 @@ +var pull = require('pull-stream') +var paramap = require('pull-paramap') +var lru = require('hashlru') +var memo = require('asyncmemo') +var u = require('./util') +var packidx = require('pull-git-packidx-parser') +var Reader = require('pull-reader') +var toPull = require('stream-to-pull-stream') +var zlib = require('zlib') + +module.exports = Git + +function Git(app) { + this.app = app + this.findObject = memo({ + cache: false, + asString: function (opts) { + return opts.id + opts.headMsgId + } + }, this._findObject.bind(this)) +} + +Git.prototype.getObject = function (opts, cb) { + pull( + this.readObject(opts), + u.pullConcat(cb) + ) +} + +// get a message that pushed an object +Git.prototype.getObjectMsg = function (opts) { + this.findObject(opts, function (err, loc) { + if (err) return cb(err) + cb(null, loc.msg) + }) +} + +Git.prototype.readObject = function (opts) { + var self = this + return u.readNext(function (cb) { + self.findObject(opts, function (err, loc) { + if (err) return cb(err) + self.app.ensureHasBlobs([loc.packLink], function (err) { + if (err) return cb(err) + cb(null, pull( + self.app.readBlob(loc.packLink, {start: loc.offset, end: loc.next}), + self.decodeObject({ + type: opts.type, + length: opts.length, + packLink: loc.packLink, + idx: loc.idx, + }) + )) + }) + }) + }) +} + +// find which packfile contains a git object, and where in the packfile it is +// located +Git.prototype._findObject = function (opts, cb) { + var self = this + var objId = opts.id + var objIdBuf = new Buffer(objId, 'hex') + self.findObjectMsgs(opts, function (err, msgs) { + if (err) return cb(err) + if (msgs.length === 0) + return cb(new Error('unable to find git object ' + objId)) + // if blobs may need to be fetched, try to ask the user about as many of them + // at one time as possible + var packidxs = [].concat.apply([], msgs.map(function (msg) { + var c = msg.value.content + var idxs = u.toArray(c.indexes).map(u.toLink) + return u.toArray(c.packs).map(u.toLink).map(function (pack, i) { + var idx = idxs[i] + if (pack && idx) return { + msg: msg, + packLink: pack, + idxLink: idx, + } + }) + })).filter(Boolean) + var blobLinks = packidxs.length === 1 + ? [packidxs[0].idxLink, packidxs[0].packLink] + : packidxs.map(function (packidx) { + return packidx.idxLink + }) + self.app.ensureHasBlobs(blobLinks, function (err) { + if (err) return cb(err) + pull( + pull.values(packidxs), + paramap(function (pack, cb) { + console.error('get idx', pack.idxLink) + self.getPackIndex(pack.idxLink, function (err, idx) { + if (err) return cb(err) + var offset = idx.find(objIdBuf) + // console.error('got idx', err, pack.idxId, offset) + if (!offset) return cb() + cb(null, { + offset: offset.offset, + next: offset.next, + packLink: pack.packLink, + idx: idx, + msg: pack.msg, + }) + }) + }, 4), + pull.filter(), + pull.take(1), + pull.collect(function (err, offsets) { + if (err) return cb(err) + if (offsets.length === 0) + return cb(new Error('unable to find git object ' + objId + + ' in ' + msgs.length + ' messages')) + cb(null, offsets[0]) + }) + ) + }) + }) +} + +// given an object id and ssb msg id, get a set of messages of which at least one pushed the object. +Git.prototype.findObjectMsgs = function (opts, cb) { + var self = this + var id = opts.id + var headMsgId = opts.headMsgId + var ended = false + var waiting = 0 + var maybeMsgs = [] + + function cbOnce(err, msgs) { + if (ended) return + ended = true + cb(err, msgs) + } + + function objectMatches(commit) { + return commit && (commit === id || commit.sha1 === id) + } + + if (!headMsgId) return cb(new TypeError('missing head message id')) + if (!u.isRef(headMsgId)) + return cb(new TypeError('bad head message id \'' + headMsgId + '\'')) + + ;(function getMsg(id) { + waiting++ + console.error('get msg', id) + self.app.getMsgDecrypted(id, function (err, msg) { + waiting-- + if (ended) return + if (err && err.name == 'NotFoundError') + return cbOnce(new Error('missing message ' + headMsgId)) + if (err) return cbOnce(err) + var c = msg.value.content + if (typeof c === 'string') + return cbOnce(new Error('unable to decrypt message ' + msg.key)) + if ((u.toArray(c.object_ids).some(objectMatches)) + || (u.toArray(c.tags).some(objectMatches)) + || (u.toArray(c.commits).some(objectMatches))) { + // found the object + return cbOnce(null, [msg]) + // console.error('found', msg.key) + } else if (!c.object_ids) { + // the object might be here + maybeMsgs.push(msg) + } + // traverse the DAG to keep looking for the object + u.toArray(c.repoBranch).filter(u.isRef).forEach(getMsg) + if (waiting === 0) { + // console.error('trying messages', maybeMsgs.map(function (msg) { return msg.key})) + cbOnce(null, maybeMsgs) + } + }) + })(headMsgId) +} + +Git.prototype.getPackIndex = function (idxBlobLink, cb) { + pull(this.app.readBlob(idxBlobLink), packidx(cb)) +} + +var objectTypes = [ + 'none', 'commit', 'tree', 'blob', + 'tag', 'unused', 'ofs-delta', 'ref-delta' +] + +function readTypedVarInt(reader, cb) { + var type, value, shift + reader.read(1, function (end, buf) { + if (ended = end) return cb(end) + var firstByte = buf[0] + type = objectTypes[(firstByte >> 4) & 7] + value = firstByte & 15 + shift = 4 + checkByte(firstByte) + }) + + function checkByte(byte) { + if (byte & 0x80) + reader.read(1, gotByte) + else + cb(null, type, value) + } + + function gotByte(end, buf) { + if (ended = end) return cb(end) + var byte = buf[0] + value += (byte & 0x7f) << shift + shift += 7 + checkByte(byte) + } +} + +function readVarInt(reader, cb) { + var value = 0, shift = 0 + reader.read(1, function gotByte(end, buf) { + if (ended = end) return cb(end) + var byte = buf[0] + value += (byte & 0x7f) << shift + shift += 7 + if (byte & 0x80) + reader.read(1, gotByte) + else + cb(null, value) + }) +} + +function inflate(read) { + return toPull(zlib.createInflate())(read) +} + +Git.prototype.decodeObject = function (opts) { + var self = this + var packLink = opts.packLink + return function (read) { + var reader = Reader() + reader(read) + return u.readNext(function (cb) { + readTypedVarInt(reader, function (end, type, length) { + if (end === true) cb(new Error('Missing object type')) + else if (end) cb(end) + else if (type === 'ref-delta') getObjectFromRefDelta(length, cb) + else if (opts.type && type !== opts.type) + cb(new Error('expected type \'' + opts.type + '\' ' + + 'but found \'' + type + '\'')) + else if (opts.length && length !== opts.length) + cb(new Error('expected length ' + opts.length + ' ' + + 'but found ' + length)) + else cb(null, inflate(reader.read())) + }) + }) + + function getObjectFromRefDelta(length, cb) { + console.error('read from ref delta') + reader.read(20, function (end, sourceHash) { + if (end) return cb(end) + var inflatedReader = Reader() + pull(reader.read(), inflate, inflatedReader) + readVarInt(inflatedReader, function (err, expectedSourceLength) { + if (err) return cb(err) + readVarInt(inflatedReader, function (err, expectedTargetLength) { + if (err) return cb(err) + // console.error('getting object', sourceHash) + var offset = opts.idx.find(sourceHash) + if (!offset) return cb(null, 'missing source object ' + + sourcehash.toString('hex')) + console.error('get pack', opts.packLink, offset.offset, offset.next) + var readSource = pull( + self.app.readBlob(opts.packLink, { + start: offset.offset, + end: offset.next + }), + self.decodeObject({ + type: opts.type, + length: expectedSourceLength, + packLink: opts.packLink, + idx: opts.idx + }) + ) + // console.error('patching', length, expectedTargetLength) + cb(null, patchObject(inflatedReader, length, readSource, expectedTargetLength)) + }) + }) + }) + } + } +} + +function readOffsetSize(cmd, reader, readCb) { + var offset = 0, size = 0 + + function addByte(bit, outPos, cb) { + if (cmd & (1 << bit)) + reader.read(1, function (err, buf) { + if (err) readCb(err) + else cb(buf[0] << (outPos << 3)) + }) + else + cb(0) + } + + addByte(0, 0, function (val) { + offset = val + addByte(1, 1, function (val) { + offset |= val + addByte(2, 2, function (val) { + offset |= val + addByte(3, 3, function (val) { + offset |= val + addSize() + }) + }) + }) + }) + function addSize() { + addByte(4, 0, function (val) { + size = val + addByte(5, 1, function (val) { + size |= val + addByte(6, 2, function (val) { + size |= val + readCb(null, offset, size || 0x10000) + }) + }) + }) + } +} + +function patchObject(deltaReader, deltaLength, readSource, targetLength) { + var srcBuf + var ended + // console.error('patching', deltaLength, targetLength) + + return u.readNext(function (cb) { + pull(readSource, u.pullConcat(function (err, buf) { + if (err) return cb(err) + srcBuf = buf + cb(null, read) + })) + }) + + function read(abort, cb) { + // console.error('pa', abort, ended) + if (ended) return cb(ended) + deltaReader.read(1, function (end, dBuf) { + // console.error("read", end, dBuf) + // if (ended = end) return console.error('patched', deltaLength, targetLength, end), cb(end) + if (ended = end) return cb(end) + var cmd = dBuf[0] + // console.error('cmd', cmd & 0x80, cmd) + if (cmd & 0x80) + // skip a variable amount and then pass through a variable amount + readOffsetSize(cmd, deltaReader, function (err, offset, size) { + // console.error('offset', err, offset, size) + if (err) return earlyEnd(err) + var buf = srcBuf.slice(offset, offset + size) + // console.error('buf', buf) + cb(end, buf) + }) + else if (cmd) + // insert `cmd` bytes from delta + deltaReader.read(cmd, cb) + else + cb(new Error("unexpected delta opcode 0")) + }) + + function earlyEnd(err) { + cb(err === true ? new Error('stream ended early') : err) + } + } +} + +Git.prototype.getCommit = function (opts, cb) { + this.getObject(opts, function (err, buf) { + if (err) return cb(err) + var commit = { + body: buf.toString('ascii') + } + cb(null, commit) + }) +} diff --git a/lib/render-msg.js b/lib/render-msg.js index c0001fc..ef23b3f 100644 --- a/lib/render-msg.js +++ b/lib/render-msg.js @@ -25,11 +25,7 @@ RenderMsg.prototype.toUrl = function (href) { } RenderMsg.prototype.linkify = function (text) { - var arr = text.split(u.ssbRefRegex) - for (var i = 1; i < arr.length; i += 2) { - arr[i] = h('a', {href: this.toUrl(arr[i])}, arr[i]) - } - return arr + return this.render.linkify(text) } function token() { @@ -482,14 +478,20 @@ RenderMsg.prototype.gitUpdate = function (cb) { !isNaN(size) ? [self.render.formatSize(size), ' '] : '', self.c.refs ? h('ul', Object.keys(self.c.refs).map(function (ref) { var id = self.c.refs[ref] - return h('li', - ref.replace(/^refs\/(heads|tags)\//, ''), ': ', - id ? h('code', id) : h('em', 'deleted')) + var type = /^refs\/tags/.test(ref) ? 'tag' : 'commit' + var path = id && ('/git/' + type + '/' + encodeURIComponent(id) + + '?head=' + encodeURIComponent(self.msg.key)) + return h('li', + ref.replace(/^refs\/(heads|tags)\//, ''), ': ', + id ? h('a', {href: self.render.toUrl(path)}, h('code', id)) + : h('em', 'deleted')) })) : '', Array.isArray(self.c.commits) ? h('ul', self.c.commits.map(function (commit) { - return h('li', - h('code', String(commit.sha1).substr(0, 8)), ' ', + var path = '/git/commit/' + encodeURIComponent(commit.sha1) + + '?head=' + encodeURIComponent(self.msg.key) + return h('li', h('a', {href: self.render.toUrl(path)}, + h('code', String(commit.sha1).substr(0, 8))), ' ', self.linkify(String(commit.title)), self.gitCommitBody(commit.body) ) diff --git a/lib/render.js b/lib/render.js index dd6d55e..02d7285 100644 --- a/lib/render.js +++ b/lib/render.js @@ -148,6 +148,14 @@ Render.prototype.formatSize = function (size) { return size.toFixed(2) + ' MB' } +Render.prototype.linkify = function (text) { + var arr = text.split(u.ssbRefRegex) + for (var i = 1; i < arr.length; i += 2) { + arr[i] = h('a', {href: this.toUrl(arr[i])}, arr[i]) + } + return arr +} + Render.prototype.toUrl = function (href) { if (!href) return href var mentions = this._mentions diff --git a/lib/serve.js b/lib/serve.js index 8d3eeba..4e8c15d 100644 --- a/lib/serve.js +++ b/lib/serve.js @@ -119,6 +119,7 @@ Serve.prototype.go = function () { else if (data.action === 'publish') self.publishJSON(next) else if (data.action === 'vote') self.publishVote(next) else if (data.action === 'contact') self.publishContact(next) + else if (data.action === 'want-blobs') self.wantBlobs(next) else next() } @@ -172,6 +173,22 @@ Serve.prototype.publishContact = function (cb) { this.publish(content, cb) } +Serve.prototype.wantBlobs = function (cb) { + var self = this + if (!self.data.blob_ids) return cb() + var ids = self.data.blob_ids.split(',') + if (!ids.every(u.isRef)) return cb(new Error('bad blob ids ' + ids.join(','))) + var done = multicb({pluck: 1}) + ids.forEach(function (id) { + self.app.sbot.blobs.want(id, done()) + }) + done(function (err) { + if (err) return cb(err) + // self.note = h('div', 'wanted blobs: ' + ids.join(', ') + '.') + cb() + }) +} + Serve.prototype.publish = function (content, cb) { var self = this var done = multicb({pluck: 1, spread: true}) @@ -257,6 +274,7 @@ Serve.prototype.path = function (url) { case '/emoji': return this.emoji(m[2]) case '/contacts': return this.contacts(m[2]) case '/about': return this.about(m[2]) + case '/git': return this.git(m[2]) } return this.respond(404, 'Not found') } @@ -1063,6 +1081,20 @@ function catchHTMLError() { } } +function catchTextError() { + return function (read) { + var ended + return function (abort, cb) { + if (ended) return cb(ended) + read(abort, function (end, data) { + if (!end || end === true) return cb(end, data) + ended = true + cb(null, end.stack + '\n') + }) + } + } +} + function styles() { return fs.readFileSync(path.join(__dirname, '../static/styles.css'), 'utf8') } @@ -1117,6 +1149,7 @@ Serve.prototype.wrapPage = function (title, searchQ) { 'published ', self.app.render.msgLink(self.publishedMsg, done()) ) : '', + // self.note, content ))) done(cb) @@ -1245,6 +1278,65 @@ Serve.prototype.wrapUserFeed = function (isScrolled, id) { }) } +Serve.prototype.git = function (url) { + var m = /^\/?([^\/]*)\/?(.*)?$/.exec(url) + switch (m[1]) { + case 'commit': return this.gitCommit(m[2]) + case 'tag': return this.gitCommit(m[2]) + case 'raw': return this.gitRaw(m[2]) + default: return this.respond(404, 'Not found') + } +} + +Serve.prototype.gitRaw = function (rev) { + var self = this + if (!/[0-9a-f]{24}/.test(rev)) { + return pull( + pull.once('\'' + rev + '\' is not a git object id'), + self.respondSink(400, {'Content-Type': 'text/plain'}) + ) + } + + var headMsgId = self.query.head + pull( + self.app.git.readObject({ + id: rev, + headMsgId: headMsgId, + }), + catchTextError(), + self.respondSink(200, {'Content-type': 'text/plain'}) + ) +} + +Serve.prototype.gitCommit = function (rev) { + var self = this + if (!/[0-9a-f]{24}/.test(rev)) { + return pull( + ph('div.error', 'rev is not a git object id'), + self.wrapPage('git'), + self.respondSink(400) + ) + } + + self.app.git.getCommit({ + id: rev, + headMsgId: self.query.head, + }, function (err, commit) { + if (err && err.name === 'BlobNotFoundError') + return self.askWantBlobs(err.links) + if (err) return pull( + pull.once(u.renderError(err).outerHTML), + self.wrapPage('git object ' + rev), + self.respondSink(400) + ) + pull( + pull.once(h('pre', self.app.render.linkify(commit.body)).outerHTML), + self.wrapPage('git object ' + rev), + self.respondSink(200) + ) + }) +} + Serve.prototype.wrapPublic = function (opts) { var self = this return u.hyperwrap(function (thread, cb) { @@ -1260,6 +1352,31 @@ Serve.prototype.wrapPublic = function (opts) { }) } +Serve.prototype.askWantBlobs = function (links) { + var self = this + pull( + ph('form', {action: '', method: 'post'}, [ + ph('section', [ + ph('h3', 'Missing blobs'), + ph('p', 'The application needs these blobs to continue:'), + ph('table', links.map(u.toLink).map(function (link) { + if (!u.isRef(link.link)) return + return ph('tr', [ + ph('td', ph('code', link.link)), + ph('td', self.app.render.formatSize(link.size)), + ]) + })), + ph('input', {type: 'hidden', name: 'action', value: 'want-blobs'}), + ph('input', {type: 'hidden', name: 'blob_ids', + value: links.map(u.linkDest).join(',')}), + ph('p', ph('input', {type: 'submit', value: 'Want Blobs'})) + ]) + ]), + self.wrapPage('missing blobs'), + self.respondSink(409) + ) +} + Serve.prototype.wrapPrivate = function (opts) { var self = this return u.hyperwrap(function (thread, cb) { diff --git a/lib/util.js b/lib/util.js index 5546716..dbefa14 100644 --- a/lib/util.js +++ b/lib/util.js @@ -62,6 +62,10 @@ u.hyperwrap = function (fn) { } } +u.toLink = function (link) { + return typeof link === 'string' ? {link: link} : link +} + u.linkDest = function (link) { return typeof link === 'string' ? link : link && link.link || link } @@ -114,3 +118,19 @@ u.isMsgEncrypted = function (msg) { var c = msg && msg.value.content return typeof c === 'string' } + +u.pullConcat = function (cb) { + return pull.collect(function (err, bufs) { + if (err) return cb(err) + cb(null, Buffer.concat(bufs)) + }) +} + +u.customError = function (name) { + return function (message) { + var error = new Error(message) + error.name = name + error.stack = error.stack.replace(/^ at .*\n/m, '') + return error + } +} diff --git a/package.json b/package.json index 7b77ca5..2a78964 100644 --- a/package.json +++ b/package.json @@ -13,11 +13,13 @@ "mime-types": "^2.1.12", "multicb": "^1.2.1", "pull-cat": "^1.1.11", + "pull-git-packidx-parser": "^1.0.0", "pull-hash": "^1.0.0", "pull-hyperscript": "^0.2.2", "pull-identify-filetype": "^1.1.0", "pull-paginate": "^1.0.0", "pull-paramap": "^1.2.1", + "pull-reader": "^1.2.9", "pull-stream": "^3.5.0", "ssb-contact": "^1.0.0", "ssb-marked": "^0.7.1", |