aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--lib/app.js32
-rw-r--r--lib/git.js380
-rw-r--r--lib/render-msg.js22
-rw-r--r--lib/render.js8
-rw-r--r--lib/serve.js117
-rw-r--r--lib/util.js20
-rw-r--r--package.json2
7 files changed, 571 insertions, 10 deletions
diff --git a/lib/app.js b/lib/app.js
index 4219151..1daf064 100644
--- a/lib/app.js
+++ b/lib/app.js
@@ -11,6 +11,7 @@ var Contacts = require('ssb-contact')
var About = require('./about')
var Serve = require('./serve')
var Render = require('./render')
+var Git = require('./git')
module.exports = App
@@ -41,6 +42,7 @@ function App(sbot, config) {
this.unboxMsg = this.unboxMsg.bind(this)
this.render = new Render(this, this.opts)
+ this.git = new Git(this)
}
App.prototype.go = function () {
@@ -186,6 +188,36 @@ App.prototype.pushBlob = function (id, cb) {
this.sbot.blobs.push(id, cb)
}
+App.prototype.readBlob = function (link, opts) {
+ link = u.toLink(link)
+ opts = opts || {}
+ return this.sbot.blobs.get({
+ hash: link.link,
+ size: link.size,
+ start: opts.start,
+ end: opts.end,
+ })
+}
+
+App.prototype.ensureHasBlobs = function (links, cb) {
+ var self = this
+ var done = multicb({pluck: 1})
+ links.forEach(function (link) {
+ var cb = done()
+ self.sbot.blobs.size(link.link, function (err, size) {
+ if (err) cb(err)
+ else if (size == null) cb(null, link)
+ else cb()
+ })
+ })
+ done(function (err, missingLinks) {
+ if (err) console.trace(err)
+ missingLinks = missingLinks.filter(Boolean)
+ if (missingLinks.length == 0) return cb()
+ return cb({name: 'BlobNotFoundError', links: missingLinks})
+ })
+}
+
App.prototype.getReverseNameSync = function (name) {
var id = this.reverseNameCache.get(name)
return id
diff --git a/lib/git.js b/lib/git.js
new file mode 100644
index 0000000..36e4587
--- /dev/null
+++ b/lib/git.js
@@ -0,0 +1,380 @@
+var pull = require('pull-stream')
+var paramap = require('pull-paramap')
+var lru = require('hashlru')
+var memo = require('asyncmemo')
+var u = require('./util')
+var packidx = require('pull-git-packidx-parser')
+var Reader = require('pull-reader')
+var toPull = require('stream-to-pull-stream')
+var zlib = require('zlib')
+
+module.exports = Git
+
+function Git(app) {
+ this.app = app
+ this.findObject = memo({
+ cache: false,
+ asString: function (opts) {
+ return opts.id + opts.headMsgId
+ }
+ }, this._findObject.bind(this))
+}
+
+Git.prototype.getObject = function (opts, cb) {
+ pull(
+ this.readObject(opts),
+ u.pullConcat(cb)
+ )
+}
+
+// get a message that pushed an object
+Git.prototype.getObjectMsg = function (opts) {
+ this.findObject(opts, function (err, loc) {
+ if (err) return cb(err)
+ cb(null, loc.msg)
+ })
+}
+
+Git.prototype.readObject = function (opts) {
+ var self = this
+ return u.readNext(function (cb) {
+ self.findObject(opts, function (err, loc) {
+ if (err) return cb(err)
+ self.app.ensureHasBlobs([loc.packLink], function (err) {
+ if (err) return cb(err)
+ cb(null, pull(
+ self.app.readBlob(loc.packLink, {start: loc.offset, end: loc.next}),
+ self.decodeObject({
+ type: opts.type,
+ length: opts.length,
+ packLink: loc.packLink,
+ idx: loc.idx,
+ })
+ ))
+ })
+ })
+ })
+}
+
+// find which packfile contains a git object, and where in the packfile it is
+// located
+Git.prototype._findObject = function (opts, cb) {
+ var self = this
+ var objId = opts.id
+ var objIdBuf = new Buffer(objId, 'hex')
+ self.findObjectMsgs(opts, function (err, msgs) {
+ if (err) return cb(err)
+ if (msgs.length === 0)
+ return cb(new Error('unable to find git object ' + objId))
+ // if blobs may need to be fetched, try to ask the user about as many of them
+ // at one time as possible
+ var packidxs = [].concat.apply([], msgs.map(function (msg) {
+ var c = msg.value.content
+ var idxs = u.toArray(c.indexes).map(u.toLink)
+ return u.toArray(c.packs).map(u.toLink).map(function (pack, i) {
+ var idx = idxs[i]
+ if (pack && idx) return {
+ msg: msg,
+ packLink: pack,
+ idxLink: idx,
+ }
+ })
+ })).filter(Boolean)
+ var blobLinks = packidxs.length === 1
+ ? [packidxs[0].idxLink, packidxs[0].packLink]
+ : packidxs.map(function (packidx) {
+ return packidx.idxLink
+ })
+ self.app.ensureHasBlobs(blobLinks, function (err) {
+ if (err) return cb(err)
+ pull(
+ pull.values(packidxs),
+ paramap(function (pack, cb) {
+ console.error('get idx', pack.idxLink)
+ self.getPackIndex(pack.idxLink, function (err, idx) {
+ if (err) return cb(err)
+ var offset = idx.find(objIdBuf)
+ // console.error('got idx', err, pack.idxId, offset)
+ if (!offset) return cb()
+ cb(null, {
+ offset: offset.offset,
+ next: offset.next,
+ packLink: pack.packLink,
+ idx: idx,
+ msg: pack.msg,
+ })
+ })
+ }, 4),
+ pull.filter(),
+ pull.take(1),
+ pull.collect(function (err, offsets) {
+ if (err) return cb(err)
+ if (offsets.length === 0)
+ return cb(new Error('unable to find git object ' + objId +
+ ' in ' + msgs.length + ' messages'))
+ cb(null, offsets[0])
+ })
+ )
+ })
+ })
+}
+
+// given an object id and ssb msg id, get a set of messages of which at least one pushed the object.
+Git.prototype.findObjectMsgs = function (opts, cb) {
+ var self = this
+ var id = opts.id
+ var headMsgId = opts.headMsgId
+ var ended = false
+ var waiting = 0
+ var maybeMsgs = []
+
+ function cbOnce(err, msgs) {
+ if (ended) return
+ ended = true
+ cb(err, msgs)
+ }
+
+ function objectMatches(commit) {
+ return commit && (commit === id || commit.sha1 === id)
+ }
+
+ if (!headMsgId) return cb(new TypeError('missing head message id'))
+ if (!u.isRef(headMsgId))
+ return cb(new TypeError('bad head message id \'' + headMsgId + '\''))
+
+ ;(function getMsg(id) {
+ waiting++
+ console.error('get msg', id)
+ self.app.getMsgDecrypted(id, function (err, msg) {
+ waiting--
+ if (ended) return
+ if (err && err.name == 'NotFoundError')
+ return cbOnce(new Error('missing message ' + headMsgId))
+ if (err) return cbOnce(err)
+ var c = msg.value.content
+ if (typeof c === 'string')
+ return cbOnce(new Error('unable to decrypt message ' + msg.key))
+ if ((u.toArray(c.object_ids).some(objectMatches))
+ || (u.toArray(c.tags).some(objectMatches))
+ || (u.toArray(c.commits).some(objectMatches))) {
+ // found the object
+ return cbOnce(null, [msg])
+ // console.error('found', msg.key)
+ } else if (!c.object_ids) {
+ // the object might be here
+ maybeMsgs.push(msg)
+ }
+ // traverse the DAG to keep looking for the object
+ u.toArray(c.repoBranch).filter(u.isRef).forEach(getMsg)
+ if (waiting === 0) {
+ // console.error('trying messages', maybeMsgs.map(function (msg) { return msg.key}))
+ cbOnce(null, maybeMsgs)
+ }
+ })
+ })(headMsgId)
+}
+
+Git.prototype.getPackIndex = function (idxBlobLink, cb) {
+ pull(this.app.readBlob(idxBlobLink), packidx(cb))
+}
+
+var objectTypes = [
+ 'none', 'commit', 'tree', 'blob',
+ 'tag', 'unused', 'ofs-delta', 'ref-delta'
+]
+
+function readTypedVarInt(reader, cb) {
+ var type, value, shift
+ reader.read(1, function (end, buf) {
+ if (ended = end) return cb(end)
+ var firstByte = buf[0]
+ type = objectTypes[(firstByte >> 4) & 7]
+ value = firstByte & 15
+ shift = 4
+ checkByte(firstByte)
+ })
+
+ function checkByte(byte) {
+ if (byte & 0x80)
+ reader.read(1, gotByte)
+ else
+ cb(null, type, value)
+ }
+
+ function gotByte(end, buf) {
+ if (ended = end) return cb(end)
+ var byte = buf[0]
+ value += (byte & 0x7f) << shift
+ shift += 7
+ checkByte(byte)
+ }
+}
+
+function readVarInt(reader, cb) {
+ var value = 0, shift = 0
+ reader.read(1, function gotByte(end, buf) {
+ if (ended = end) return cb(end)
+ var byte = buf[0]
+ value += (byte & 0x7f) << shift
+ shift += 7
+ if (byte & 0x80)
+ reader.read(1, gotByte)
+ else
+ cb(null, value)
+ })
+}
+
+function inflate(read) {
+ return toPull(zlib.createInflate())(read)
+}
+
+Git.prototype.decodeObject = function (opts) {
+ var self = this
+ var packLink = opts.packLink
+ return function (read) {
+ var reader = Reader()
+ reader(read)
+ return u.readNext(function (cb) {
+ readTypedVarInt(reader, function (end, type, length) {
+ if (end === true) cb(new Error('Missing object type'))
+ else if (end) cb(end)
+ else if (type === 'ref-delta') getObjectFromRefDelta(length, cb)
+ else if (opts.type && type !== opts.type)
+ cb(new Error('expected type \'' + opts.type + '\' ' +
+ 'but found \'' + type + '\''))
+ else if (opts.length && length !== opts.length)
+ cb(new Error('expected length ' + opts.length + ' ' +
+ 'but found ' + length))
+ else cb(null, inflate(reader.read()))
+ })
+ })
+
+ function getObjectFromRefDelta(length, cb) {
+ console.error('read from ref delta')
+ reader.read(20, function (end, sourceHash) {
+ if (end) return cb(end)
+ var inflatedReader = Reader()
+ pull(reader.read(), inflate, inflatedReader)
+ readVarInt(inflatedReader, function (err, expectedSourceLength) {
+ if (err) return cb(err)
+ readVarInt(inflatedReader, function (err, expectedTargetLength) {
+ if (err) return cb(err)
+ // console.error('getting object', sourceHash)
+ var offset = opts.idx.find(sourceHash)
+ if (!offset) return cb(null, 'missing source object ' +
+ sourcehash.toString('hex'))
+ console.error('get pack', opts.packLink, offset.offset, offset.next)
+ var readSource = pull(
+ self.app.readBlob(opts.packLink, {
+ start: offset.offset,
+ end: offset.next
+ }),
+ self.decodeObject({
+ type: opts.type,
+ length: expectedSourceLength,
+ packLink: opts.packLink,
+ idx: opts.idx
+ })
+ )
+ // console.error('patching', length, expectedTargetLength)
+ cb(null, patchObject(inflatedReader, length, readSource, expectedTargetLength))
+ })
+ })
+ })
+ }
+ }
+}
+
+function readOffsetSize(cmd, reader, readCb) {
+ var offset = 0, size = 0
+
+ function addByte(bit, outPos, cb) {
+ if (cmd & (1 << bit))
+ reader.read(1, function (err, buf) {
+ if (err) readCb(err)
+ else cb(buf[0] << (outPos << 3))
+ })
+ else
+ cb(0)
+ }
+
+ addByte(0, 0, function (val) {
+ offset = val
+ addByte(1, 1, function (val) {
+ offset |= val
+ addByte(2, 2, function (val) {
+ offset |= val
+ addByte(3, 3, function (val) {
+ offset |= val
+ addSize()
+ })
+ })
+ })
+ })
+ function addSize() {
+ addByte(4, 0, function (val) {
+ size = val
+ addByte(5, 1, function (val) {
+ size |= val
+ addByte(6, 2, function (val) {
+ size |= val
+ readCb(null, offset, size || 0x10000)
+ })
+ })
+ })
+ }
+}
+
+function patchObject(deltaReader, deltaLength, readSource, targetLength) {
+ var srcBuf
+ var ended
+ // console.error('patching', deltaLength, targetLength)
+
+ return u.readNext(function (cb) {
+ pull(readSource, u.pullConcat(function (err, buf) {
+ if (err) return cb(err)
+ srcBuf = buf
+ cb(null, read)
+ }))
+ })
+
+ function read(abort, cb) {
+ // console.error('pa', abort, ended)
+ if (ended) return cb(ended)
+ deltaReader.read(1, function (end, dBuf) {
+ // console.error("read", end, dBuf)
+ // if (ended = end) return console.error('patched', deltaLength, targetLength, end), cb(end)
+ if (ended = end) return cb(end)
+ var cmd = dBuf[0]
+ // console.error('cmd', cmd & 0x80, cmd)
+ if (cmd & 0x80)
+ // skip a variable amount and then pass through a variable amount
+ readOffsetSize(cmd, deltaReader, function (err, offset, size) {
+ // console.error('offset', err, offset, size)
+ if (err) return earlyEnd(err)
+ var buf = srcBuf.slice(offset, offset + size)
+ // console.error('buf', buf)
+ cb(end, buf)
+ })
+ else if (cmd)
+ // insert `cmd` bytes from delta
+ deltaReader.read(cmd, cb)
+ else
+ cb(new Error("unexpected delta opcode 0"))
+ })
+
+ function earlyEnd(err) {
+ cb(err === true ? new Error('stream ended early') : err)
+ }
+ }
+}
+
+Git.prototype.getCommit = function (opts, cb) {
+ this.getObject(opts, function (err, buf) {
+ if (err) return cb(err)
+ var commit = {
+ body: buf.toString('ascii')
+ }
+ cb(null, commit)
+ })
+}
diff --git a/lib/render-msg.js b/lib/render-msg.js
index c0001fc..ef23b3f 100644
--- a/lib/render-msg.js
+++ b/lib/render-msg.js
@@ -25,11 +25,7 @@ RenderMsg.prototype.toUrl = function (href) {
}
RenderMsg.prototype.linkify = function (text) {
- var arr = text.split(u.ssbRefRegex)
- for (var i = 1; i < arr.length; i += 2) {
- arr[i] = h('a', {href: this.toUrl(arr[i])}, arr[i])
- }
- return arr
+ return this.render.linkify(text)
}
function token() {
@@ -482,14 +478,20 @@ RenderMsg.prototype.gitUpdate = function (cb) {
!isNaN(size) ? [self.render.formatSize(size), ' '] : '',
self.c.refs ? h('ul', Object.keys(self.c.refs).map(function (ref) {
var id = self.c.refs[ref]
- return h('li',
- ref.replace(/^refs\/(heads|tags)\//, ''), ': ',
- id ? h('code', id) : h('em', 'deleted'))
+ var type = /^refs\/tags/.test(ref) ? 'tag' : 'commit'
+ var path = id && ('/git/' + type + '/' + encodeURIComponent(id)
+ + '?head=' + encodeURIComponent(self.msg.key))
+ return h('li',
+ ref.replace(/^refs\/(heads|tags)\//, ''), ': ',
+ id ? h('a', {href: self.render.toUrl(path)}, h('code', id))
+ : h('em', 'deleted'))
})) : '',
Array.isArray(self.c.commits) ?
h('ul', self.c.commits.map(function (commit) {
- return h('li',
- h('code', String(commit.sha1).substr(0, 8)), ' ',
+ var path = '/git/commit/' + encodeURIComponent(commit.sha1)
+ + '?head=' + encodeURIComponent(self.msg.key)
+ return h('li', h('a', {href: self.render.toUrl(path)},
+ h('code', String(commit.sha1).substr(0, 8))), ' ',
self.linkify(String(commit.title)),
self.gitCommitBody(commit.body)
)
diff --git a/lib/render.js b/lib/render.js
index dd6d55e..02d7285 100644
--- a/lib/render.js
+++ b/lib/render.js
@@ -148,6 +148,14 @@ Render.prototype.formatSize = function (size) {
return size.toFixed(2) + ' MB'
}
+Render.prototype.linkify = function (text) {
+ var arr = text.split(u.ssbRefRegex)
+ for (var i = 1; i < arr.length; i += 2) {
+ arr[i] = h('a', {href: this.toUrl(arr[i])}, arr[i])
+ }
+ return arr
+}
+
Render.prototype.toUrl = function (href) {
if (!href) return href
var mentions = this._mentions
diff --git a/lib/serve.js b/lib/serve.js
index 8d3eeba..4e8c15d 100644
--- a/lib/serve.js
+++ b/lib/serve.js
@@ -119,6 +119,7 @@ Serve.prototype.go = function () {
else if (data.action === 'publish') self.publishJSON(next)
else if (data.action === 'vote') self.publishVote(next)
else if (data.action === 'contact') self.publishContact(next)
+ else if (data.action === 'want-blobs') self.wantBlobs(next)
else next()
}
@@ -172,6 +173,22 @@ Serve.prototype.publishContact = function (cb) {
this.publish(content, cb)
}
+Serve.prototype.wantBlobs = function (cb) {
+ var self = this
+ if (!self.data.blob_ids) return cb()
+ var ids = self.data.blob_ids.split(',')
+ if (!ids.every(u.isRef)) return cb(new Error('bad blob ids ' + ids.join(',')))
+ var done = multicb({pluck: 1})
+ ids.forEach(function (id) {
+ self.app.sbot.blobs.want(id, done())
+ })
+ done(function (err) {
+ if (err) return cb(err)
+ // self.note = h('div', 'wanted blobs: ' + ids.join(', ') + '.')
+ cb()
+ })
+}
+
Serve.prototype.publish = function (content, cb) {
var self = this
var done = multicb({pluck: 1, spread: true})
@@ -257,6 +274,7 @@ Serve.prototype.path = function (url) {
case '/emoji': return this.emoji(m[2])
case '/contacts': return this.contacts(m[2])
case '/about': return this.about(m[2])
+ case '/git': return this.git(m[2])
}
return this.respond(404, 'Not found')
}
@@ -1063,6 +1081,20 @@ function catchHTMLError() {
}
}
+function catchTextError() {
+ return function (read) {
+ var ended
+ return function (abort, cb) {
+ if (ended) return cb(ended)
+ read(abort, function (end, data) {
+ if (!end || end === true) return cb(end, data)
+ ended = true
+ cb(null, end.stack + '\n')
+ })
+ }
+ }
+}
+
function styles() {
return fs.readFileSync(path.join(__dirname, '../static/styles.css'), 'utf8')
}
@@ -1117,6 +1149,7 @@ Serve.prototype.wrapPage = function (title, searchQ) {
'published ',
self.app.render.msgLink(self.publishedMsg, done())
) : '',
+ // self.note,
content
)))
done(cb)
@@ -1245,6 +1278,65 @@ Serve.prototype.wrapUserFeed = function (isScrolled, id) {
})
}
+Serve.prototype.git = function (url) {
+ var m = /^\/?([^\/]*)\/?(.*)?$/.exec(url)
+ switch (m[1]) {
+ case 'commit': return this.gitCommit(m[2])
+ case 'tag': return this.gitCommit(m[2])
+ case 'raw': return this.gitRaw(m[2])
+ default: return this.respond(404, 'Not found')
+ }
+}
+
+Serve.prototype.gitRaw = function (rev) {
+ var self = this
+ if (!/[0-9a-f]{24}/.test(rev)) {
+ return pull(
+ pull.once('\'' + rev + '\' is not a git object id'),
+ self.respondSink(400, {'Content-Type': 'text/plain'})
+ )
+ }
+
+ var headMsgId = self.query.head
+ pull(
+ self.app.git.readObject({
+ id: rev,
+ headMsgId: headMsgId,
+ }),
+ catchTextError(),
+ self.respondSink(200, {'Content-type': 'text/plain'})
+ )
+}
+
+Serve.prototype.gitCommit = function (rev) {
+ var self = this
+ if (!/[0-9a-f]{24}/.test(rev)) {
+ return pull(
+ ph('div.error', 'rev is not a git object id'),
+ self.wrapPage('git'),
+ self.respondSink(400)
+ )
+ }
+
+ self.app.git.getCommit({
+ id: rev,
+ headMsgId: self.query.head,
+ }, function (err, commit) {
+ if (err && err.name === 'BlobNotFoundError')
+ return self.askWantBlobs(err.links)
+ if (err) return pull(
+ pull.once(u.renderError(err).outerHTML),
+ self.wrapPage('git object ' + rev),
+ self.respondSink(400)
+ )
+ pull(
+ pull.once(h('pre', self.app.render.linkify(commit.body)).outerHTML),
+ self.wrapPage('git object ' + rev),
+ self.respondSink(200)
+ )
+ })
+}
+
Serve.prototype.wrapPublic = function (opts) {
var self = this
return u.hyperwrap(function (thread, cb) {
@@ -1260,6 +1352,31 @@ Serve.prototype.wrapPublic = function (opts) {
})
}
+Serve.prototype.askWantBlobs = function (links) {
+ var self = this
+ pull(
+ ph('form', {action: '', method: 'post'}, [
+ ph('section', [
+ ph('h3', 'Missing blobs'),
+ ph('p', 'The application needs these blobs to continue:'),
+ ph('table', links.map(u.toLink).map(function (link) {
+ if (!u.isRef(link.link)) return
+ return ph('tr', [
+ ph('td', ph('code', link.link)),
+ ph('td', self.app.render.formatSize(link.size)),
+ ])
+ })),
+ ph('input', {type: 'hidden', name: 'action', value: 'want-blobs'}),
+ ph('input', {type: 'hidden', name: 'blob_ids',
+ value: links.map(u.linkDest).join(',')}),
+ ph('p', ph('input', {type: 'submit', value: 'Want Blobs'}))
+ ])
+ ]),
+ self.wrapPage('missing blobs'),
+ self.respondSink(409)
+ )
+}
+
Serve.prototype.wrapPrivate = function (opts) {
var self = this
return u.hyperwrap(function (thread, cb) {
diff --git a/lib/util.js b/lib/util.js
index 5546716..dbefa14 100644
--- a/lib/util.js
+++ b/lib/util.js
@@ -62,6 +62,10 @@ u.hyperwrap = function (fn) {
}
}
+u.toLink = function (link) {
+ return typeof link === 'string' ? {link: link} : link
+}
+
u.linkDest = function (link) {
return typeof link === 'string' ? link : link && link.link || link
}
@@ -114,3 +118,19 @@ u.isMsgEncrypted = function (msg) {
var c = msg && msg.value.content
return typeof c === 'string'
}
+
+u.pullConcat = function (cb) {
+ return pull.collect(function (err, bufs) {
+ if (err) return cb(err)
+ cb(null, Buffer.concat(bufs))
+ })
+}
+
+u.customError = function (name) {
+ return function (message) {
+ var error = new Error(message)
+ error.name = name
+ error.stack = error.stack.replace(/^ at .*\n/m, '')
+ return error
+ }
+}
diff --git a/package.json b/package.json
index 7b77ca5..2a78964 100644
--- a/package.json
+++ b/package.json
@@ -13,11 +13,13 @@
"mime-types": "^2.1.12",
"multicb": "^1.2.1",
"pull-cat": "^1.1.11",
+ "pull-git-packidx-parser": "^1.0.0",
"pull-hash": "^1.0.0",
"pull-hyperscript": "^0.2.2",
"pull-identify-filetype": "^1.1.0",
"pull-paginate": "^1.0.0",
"pull-paramap": "^1.2.1",
+ "pull-reader": "^1.2.9",
"pull-stream": "^3.5.0",
"ssb-contact": "^1.0.0",
"ssb-marked": "^0.7.1",