From 7570f001cc8ddeff88ebdab640e06a0b1b03974c Mon Sep 17 00:00:00 2001 From: cel Date: Thu, 4 May 2017 11:03:16 -1000 Subject: wip: about --- lib/app.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib/app.js') diff --git a/lib/app.js b/lib/app.js index a76687c..a3bfbf6 100644 --- a/lib/app.js +++ b/lib/app.js @@ -9,7 +9,7 @@ var hasher = require('pull-hash/ext/ssb') var multicb = require('multicb') var paramap = require('pull-paramap') var Contacts = require('ssb-contact') - +var About = require('./about') var Serve = require('./serve') var Render = require('./render') @@ -280,3 +280,7 @@ App.prototype.streamChannels = function (opts) { App.prototype.createContactStreams = function (id) { return new Contacts(this.sbot).createContactStreams(id) } + +App.prototype.createAboutStreams = function (id) { + return new About(this.sbot).createAboutStreams(id) +} -- cgit v1.2.3 From 1e1d32ef170391da41c2c064fc9a557a1ba667d1 Mon Sep 17 00:00:00 2001 From: _ssb <_ssb@trouble.kagu-tsuchi.com> Date: Wed, 10 May 2017 22:06:25 -0500 Subject: First pass at listing subscribed channels --- lib/app.js | 16 +++++++++++++++ lib/serve.js | 66 +++++++++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 70 insertions(+), 12 deletions(-) (limited to 'lib/app.js') diff --git a/lib/app.js b/lib/app.js index b57e5a6..f475bd2 100644 --- a/lib/app.js +++ b/lib/app.js @@ -282,6 +282,22 @@ App.prototype.streamChannels = function (opts) { ) } +App.prototype.streamMyChannels = function (id, opts) { + return pull( + this.sbot.createUserStream({id: id, reverse: true}), + this.unboxMessages(), + pull.filter(function (msg) { + if (msg.value.content.type == 'channel') { + return msg.value.content.subscribed + } + }), + pull.map(function (msg) { + return msg.value.content.channel + }), + pull.unique() + ) +} + App.prototype.createContactStreams = function (id) { return new Contacts(this.sbot).createContactStreams(id) } diff --git a/lib/serve.js b/lib/serve.js index defac87..54516e1 100644 --- a/lib/serve.js +++ b/lib/serve.js @@ -543,20 +543,48 @@ Serve.prototype.peers = function (ext) { Serve.prototype.channels = function (ext) { var self = this + var id = self.app.sbot.id + + function renderMyChannels() { + return pull( + self.app.streamMyChannels(id), + paramap(function (channel, cb) { + // var subscribed = false + cb(null, [ + h('a', {href: self.app.render.toUrl('/channel/' + channel)}, '#' + channel), + ' ' + ]) + }, 8), + pull.map(u.toHTML), + self.wrapMyChannels() + ) + } + + function renderNetworkChannels() { + return pull( + self.app.streamChannels(), + paramap(function (channel, cb) { + // var subscribed = false + cb(null, [ + h('a', {href: self.app.render.toUrl('/channel/' + channel)}, '#' + channel), + ' ' + ]) + }, 8), + pull.map(u.toHTML), + self.wrapChannels() + ) + } pull( - self.app.streamChannels(), - paramap(function (channel, cb) { - var subscribed = false - cb(null, [ - h('a', {href: self.app.render.toUrl('/channel/' + channel)}, '#' + channel), - ' ' + cat([ + ph('section', {}, [ + ph('h3', {}, 'Channels:'), + renderMyChannels(), + renderNetworkChannels() ]) - }, 8), - pull.map(u.toHTML), - self.wrapChannels(), - self.wrapPage('channels'), - self.respondSink(200, { + ]), + this.wrapPage('channels'), + this.respondSink(200, { 'Content-Type': ctype(ext) }) ) @@ -1294,7 +1322,21 @@ Serve.prototype.wrapChannels = function (opts) { return u.hyperwrap(function (channels, cb) { cb(null, [ h('section', - h('h3', 'Channels') + h('h4', 'Network') + ), + h('section', + channels + ) + ]) + }) +} + +Serve.prototype.wrapMyChannels = function (opts) { + var self = this + return u.hyperwrap(function (channels, cb) { + cb(null, [ + h('section', + h('h4', 'Subscribed') ), h('section', channels -- cgit v1.2.3 From 24417ffb7f896971664053a13101154e40d6adce Mon Sep 17 00:00:00 2001 From: cel Date: Thu, 11 May 2017 20:25:59 -1000 Subject: Optimize streaming my channels --- lib/app.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'lib/app.js') diff --git a/lib/app.js b/lib/app.js index f475bd2..6990aa5 100644 --- a/lib/app.js +++ b/lib/app.js @@ -283,6 +283,24 @@ App.prototype.streamChannels = function (opts) { } App.prototype.streamMyChannels = function (id, opts) { + // use ssb-query plugin if it is available, since it has an index for + // author + type + if (this.sbot.query) return pull( + this.sbot.query.read({ + reverse: true, + query: [ + {$filter: { + value: { + author: id, + content: {type: 'channel', subscribed: true} + } + }}, + {$map: ['value', 'content', 'channel']} + ] + }), + pull.unique() + ) + return pull( this.sbot.createUserStream({id: id, reverse: true}), this.unboxMessages(), -- cgit v1.2.3 From e0f8badf71a127ef2f66025af6c464abe512e8a4 Mon Sep 17 00:00:00 2001 From: cel Date: Mon, 15 May 2017 13:08:31 -1000 Subject: Use own About implementation Use most popular about info instead of most recent. Slightly prefer a feed's own about info for itself. --- lib/about.js | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++------ lib/app.js | 17 +++++++++------ package.json | 1 - 3 files changed, 72 insertions(+), 13 deletions(-) (limited to 'lib/app.js') diff --git a/lib/about.js b/lib/about.js index 4f1d582..fde5e04 100644 --- a/lib/about.js +++ b/lib/about.js @@ -1,19 +1,18 @@ var pull = require('pull-stream') -// var defer = require('pull-defer') -// var many = require('pull-many') var multicb = require('multicb') +var cat = require('pull-cat') var u = require('./util') module.exports = About -function About(sbot) { - if (!(this instanceof About)) return new About(sbot) - this.sbot = sbot +function About(app, myId) { + this.app = app + this.myId = myId } About.prototype.createAboutOpStream = function (id) { return pull( - this.sbot.links({dest: id, rel: 'about', values: true, reverse: true}), + this.app.sbot.links({dest: id, rel: 'about', values: true, reverse: true}), pull.map(function (msg) { var c = msg.value.content || {} return Object.keys(c).filter(function (key) { @@ -57,3 +56,59 @@ About.prototype.createAboutStreams = function (id) { sets: u.readNext(setsDone) } } + +function computeTopAbout(aboutByFeed) { + var propValueCounts = {/* prop: {value: count} */} + var topValues = {/* prop: value */} + var topValueCounts = {/* prop: count */} + for (var feed in aboutByFeed) { + var feedAbout = aboutByFeed[feed] + for (var prop in feedAbout) { + var value = feedAbout[prop] + var valueCounts = propValueCounts[prop] || (propValueCounts[prop] = {}) + var count = (valueCounts[value] || 0) + 1 + valueCounts[value] = count + if (count > (topValueCounts[prop] || 0)) { + topValueCounts[prop] = count + topValues[prop] = value + } + } + } + return topValues +} + +About.prototype.get = function (dest, cb) { + var self = this + var aboutByFeed = {} + pull( + cat([ + dest[0] === '%' && self.app.pullGetMsg(dest), + self.app.sbot.links({ + rel: 'about', + dest: dest, + values: true, + }) + ]), + self.app.unboxMessages(), + pull.drain(function (msg) { + var author = msg.value.author + var c = msg.value.content + if (!c) return + var about = aboutByFeed[author] || (aboutByFeed[author] = {}) + if (c.name) about.name = c.name + if (c.image) about.image = u.linkDest(c.image) + if (c.description) about.description = c.description + }, function (err) { + if (err) return cb(err) + // bias the author's choices by giving them an extra vote + aboutByFeed._author = aboutByFeed[dest] + var about = {} + var myAbout = aboutByFeed[this.myId] || {} + var topAbout = computeTopAbout(aboutByFeed) + for (var k in topAbout) about[k] = topAbout[k] + // always prefer own choices + for (var k in myAbout) about[k] = myAbout[k] + cb(null, about) + }) + ) +} diff --git a/lib/app.js b/lib/app.js index dfec6bd..eee6c95 100644 --- a/lib/app.js +++ b/lib/app.js @@ -4,7 +4,6 @@ var lru = require('hashlru') var pkg = require('../package') var u = require('./util') var pull = require('pull-stream') -var ssbAvatar = require('ssb-avatar') var hasher = require('pull-hash/ext/ssb') var multicb = require('multicb') var paramap = require('pull-paramap') @@ -32,9 +31,10 @@ function App(sbot, config) { } sbot.get = memo({cache: lru(100)}, sbot.get) + this.about = new About(this, sbot.id) this.getMsg = memo({cache: lru(100)}, getMsgWithValue, sbot) this.getAbout = memo({cache: this.aboutCache = lru(500)}, - getAbout.bind(this), sbot, sbot.id) + this._getAbout.bind(this)) this.unboxContent = memo({cache: lru(100)}, sbot.private.unbox) this.reverseNameCache = lru(100) @@ -199,11 +199,12 @@ function getMsgWithValue(sbot, id, cb) { }) } -function getAbout(sbot, src, id, cb) { +App.prototype._getAbout = function (id, cb) { var self = this - ssbAvatar(sbot, src, id, function (err, about) { + if (!u.isRef(id)) return cb(null, {}) + self.about.get(id, function (err, about) { if (err) return cb(err) - var sigil = id && id[0] || '@' + var sigil = id[0] || '@' if (about.name && about.name[0] !== sigil) { about.name = sigil + about.name } @@ -212,6 +213,10 @@ function getAbout(sbot, src, id, cb) { }) } +App.prototype.pullGetMsg = function (id) { + return pull.asyncMap(this.getMsg)(pull.once(id)) +} + App.prototype.createLogStream = function (opts) { opts = opts || {} return opts.sortByTimestamp @@ -321,5 +326,5 @@ App.prototype.createContactStreams = function (id) { } App.prototype.createAboutStreams = function (id) { - return new About(this.sbot).createAboutStreams(id) + return this.about.createAboutStreams(id) } diff --git a/package.json b/package.json index 1bc5812..7b77ca5 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,6 @@ "pull-paginate": "^1.0.0", "pull-paramap": "^1.2.1", "pull-stream": "^3.5.0", - "ssb-avatar": "^0.2.0", "ssb-contact": "^1.0.0", "ssb-marked": "^0.7.1", "ssb-mentions": "^0.2.0", -- cgit v1.2.3 From b90b36f54546b4bad7618a57e7f5484892e23b52 Mon Sep 17 00:00:00 2001 From: cel Date: Tue, 16 May 2017 08:21:42 -1000 Subject: Show local name in title for links to ids as suggested in %O4GCxFpucC3DeSTCuiR9GCha9g/NXbrQYm9iam5z1RE=.sha256 --- lib/app.js | 5 +++++ lib/render.js | 27 ++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) (limited to 'lib/app.js') diff --git a/lib/app.js b/lib/app.js index eee6c95..7b5bccc 100644 --- a/lib/app.js +++ b/lib/app.js @@ -191,6 +191,11 @@ App.prototype.getReverseNameSync = function (name) { return id } +App.prototype.getNameSync = function (name) { + var about = this.aboutCache.get(name) + return about && about.name +} + function getMsgWithValue(sbot, id, cb) { if (!id) return cb() sbot.get(id, function (err, value) { diff --git a/lib/render.js b/lib/render.js index 74b4f4f..dd6d55e 100644 --- a/lib/render.js +++ b/lib/render.js @@ -34,17 +34,34 @@ MdRenderer.prototype.image = function (ref, title, text) { }).outerHTML } -MdRenderer.prototype.link = function(href, title, text) { - href = this.urltransform(href) +MdRenderer.prototype.link = function (ref, title, text) { + var href = this.urltransform(ref) var name = href && /^\/(&|%26)/.test(href) && (title || text) - return h('a', { + if (u.isRef(ref)) { + var myName = this.render.app.getNameSync(ref) + if (myName) title = title ? title + ' (' + myName + ')' : myName + } + var a = h('a', { class: href === false ? 'bad' : undefined, href: href !== false ? href : undefined, title: title || undefined, download: name ? encodeURIComponent(name) : undefined - }, text).outerHTML -}; + }) + // text is already html-escaped + a.innerHTML = text + return a.outerHTML +} +MdRenderer.prototype.mention = function (preceding, id) { + var href = this.urltransform(id) + var myName = this.render.app.getNameSync(id) + if (id.length > 50) id = id.slice(0, 8) + '…' + return (preceding||'') + h('a', { + class: href === false ? 'bad' : undefined, + href: href !== false ? href : undefined, + title: myName || undefined, + }, id).outerHTML +} function lexerRenderEmoji(emoji) { var el = this.renderer.render.emoji(emoji) -- cgit v1.2.3 From 18de89483564e9cc8d76105efa9aaf7a71c8771e Mon Sep 17 00:00:00 2001 From: cel Date: Sat, 20 May 2017 13:23:20 -1000 Subject: Increase size of reverse name cache --- lib/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/app.js') diff --git a/lib/app.js b/lib/app.js index 7b5bccc..4219151 100644 --- a/lib/app.js +++ b/lib/app.js @@ -36,7 +36,7 @@ function App(sbot, config) { this.getAbout = memo({cache: this.aboutCache = lru(500)}, this._getAbout.bind(this)) this.unboxContent = memo({cache: lru(100)}, sbot.private.unbox) - this.reverseNameCache = lru(100) + this.reverseNameCache = lru(500) this.unboxMsg = this.unboxMsg.bind(this) -- cgit v1.2.3 From d10fc1c7fbc410ec0c4773902251255582e33adc Mon Sep 17 00:00:00 2001 From: cel Date: Tue, 23 May 2017 23:24:25 -1000 Subject: wip: more git --- lib/app.js | 32 +++++ lib/git.js | 380 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/render-msg.js | 22 ++-- lib/render.js | 8 ++ lib/serve.js | 117 +++++++++++++++++ lib/util.js | 20 +++ package.json | 2 + 7 files changed, 571 insertions(+), 10 deletions(-) create mode 100644 lib/git.js (limited to 'lib/app.js') diff --git a/lib/app.js b/lib/app.js index 4219151..1daf064 100644 --- a/lib/app.js +++ b/lib/app.js @@ -11,6 +11,7 @@ var Contacts = require('ssb-contact') var About = require('./about') var Serve = require('./serve') var Render = require('./render') +var Git = require('./git') module.exports = App @@ -41,6 +42,7 @@ function App(sbot, config) { this.unboxMsg = this.unboxMsg.bind(this) this.render = new Render(this, this.opts) + this.git = new Git(this) } App.prototype.go = function () { @@ -186,6 +188,36 @@ App.prototype.pushBlob = function (id, cb) { this.sbot.blobs.push(id, cb) } +App.prototype.readBlob = function (link, opts) { + link = u.toLink(link) + opts = opts || {} + return this.sbot.blobs.get({ + hash: link.link, + size: link.size, + start: opts.start, + end: opts.end, + }) +} + +App.prototype.ensureHasBlobs = function (links, cb) { + var self = this + var done = multicb({pluck: 1}) + links.forEach(function (link) { + var cb = done() + self.sbot.blobs.size(link.link, function (err, size) { + if (err) cb(err) + else if (size == null) cb(null, link) + else cb() + }) + }) + done(function (err, missingLinks) { + if (err) console.trace(err) + missingLinks = missingLinks.filter(Boolean) + if (missingLinks.length == 0) return cb() + return cb({name: 'BlobNotFoundError', links: missingLinks}) + }) +} + App.prototype.getReverseNameSync = function (name) { var id = this.reverseNameCache.get(name) return id diff --git a/lib/git.js b/lib/git.js new file mode 100644 index 0000000..36e4587 --- /dev/null +++ b/lib/git.js @@ -0,0 +1,380 @@ +var pull = require('pull-stream') +var paramap = require('pull-paramap') +var lru = require('hashlru') +var memo = require('asyncmemo') +var u = require('./util') +var packidx = require('pull-git-packidx-parser') +var Reader = require('pull-reader') +var toPull = require('stream-to-pull-stream') +var zlib = require('zlib') + +module.exports = Git + +function Git(app) { + this.app = app + this.findObject = memo({ + cache: false, + asString: function (opts) { + return opts.id + opts.headMsgId + } + }, this._findObject.bind(this)) +} + +Git.prototype.getObject = function (opts, cb) { + pull( + this.readObject(opts), + u.pullConcat(cb) + ) +} + +// get a message that pushed an object +Git.prototype.getObjectMsg = function (opts) { + this.findObject(opts, function (err, loc) { + if (err) return cb(err) + cb(null, loc.msg) + }) +} + +Git.prototype.readObject = function (opts) { + var self = this + return u.readNext(function (cb) { + self.findObject(opts, function (err, loc) { + if (err) return cb(err) + self.app.ensureHasBlobs([loc.packLink], function (err) { + if (err) return cb(err) + cb(null, pull( + self.app.readBlob(loc.packLink, {start: loc.offset, end: loc.next}), + self.decodeObject({ + type: opts.type, + length: opts.length, + packLink: loc.packLink, + idx: loc.idx, + }) + )) + }) + }) + }) +} + +// find which packfile contains a git object, and where in the packfile it is +// located +Git.prototype._findObject = function (opts, cb) { + var self = this + var objId = opts.id + var objIdBuf = new Buffer(objId, 'hex') + self.findObjectMsgs(opts, function (err, msgs) { + if (err) return cb(err) + if (msgs.length === 0) + return cb(new Error('unable to find git object ' + objId)) + // if blobs may need to be fetched, try to ask the user about as many of them + // at one time as possible + var packidxs = [].concat.apply([], msgs.map(function (msg) { + var c = msg.value.content + var idxs = u.toArray(c.indexes).map(u.toLink) + return u.toArray(c.packs).map(u.toLink).map(function (pack, i) { + var idx = idxs[i] + if (pack && idx) return { + msg: msg, + packLink: pack, + idxLink: idx, + } + }) + })).filter(Boolean) + var blobLinks = packidxs.length === 1 + ? [packidxs[0].idxLink, packidxs[0].packLink] + : packidxs.map(function (packidx) { + return packidx.idxLink + }) + self.app.ensureHasBlobs(blobLinks, function (err) { + if (err) return cb(err) + pull( + pull.values(packidxs), + paramap(function (pack, cb) { + console.error('get idx', pack.idxLink) + self.getPackIndex(pack.idxLink, function (err, idx) { + if (err) return cb(err) + var offset = idx.find(objIdBuf) + // console.error('got idx', err, pack.idxId, offset) + if (!offset) return cb() + cb(null, { + offset: offset.offset, + next: offset.next, + packLink: pack.packLink, + idx: idx, + msg: pack.msg, + }) + }) + }, 4), + pull.filter(), + pull.take(1), + pull.collect(function (err, offsets) { + if (err) return cb(err) + if (offsets.length === 0) + return cb(new Error('unable to find git object ' + objId + + ' in ' + msgs.length + ' messages')) + cb(null, offsets[0]) + }) + ) + }) + }) +} + +// given an object id and ssb msg id, get a set of messages of which at least one pushed the object. +Git.prototype.findObjectMsgs = function (opts, cb) { + var self = this + var id = opts.id + var headMsgId = opts.headMsgId + var ended = false + var waiting = 0 + var maybeMsgs = [] + + function cbOnce(err, msgs) { + if (ended) return + ended = true + cb(err, msgs) + } + + function objectMatches(commit) { + return commit && (commit === id || commit.sha1 === id) + } + + if (!headMsgId) return cb(new TypeError('missing head message id')) + if (!u.isRef(headMsgId)) + return cb(new TypeError('bad head message id \'' + headMsgId + '\'')) + + ;(function getMsg(id) { + waiting++ + console.error('get msg', id) + self.app.getMsgDecrypted(id, function (err, msg) { + waiting-- + if (ended) return + if (err && err.name == 'NotFoundError') + return cbOnce(new Error('missing message ' + headMsgId)) + if (err) return cbOnce(err) + var c = msg.value.content + if (typeof c === 'string') + return cbOnce(new Error('unable to decrypt message ' + msg.key)) + if ((u.toArray(c.object_ids).some(objectMatches)) + || (u.toArray(c.tags).some(objectMatches)) + || (u.toArray(c.commits).some(objectMatches))) { + // found the object + return cbOnce(null, [msg]) + // console.error('found', msg.key) + } else if (!c.object_ids) { + // the object might be here + maybeMsgs.push(msg) + } + // traverse the DAG to keep looking for the object + u.toArray(c.repoBranch).filter(u.isRef).forEach(getMsg) + if (waiting === 0) { + // console.error('trying messages', maybeMsgs.map(function (msg) { return msg.key})) + cbOnce(null, maybeMsgs) + } + }) + })(headMsgId) +} + +Git.prototype.getPackIndex = function (idxBlobLink, cb) { + pull(this.app.readBlob(idxBlobLink), packidx(cb)) +} + +var objectTypes = [ + 'none', 'commit', 'tree', 'blob', + 'tag', 'unused', 'ofs-delta', 'ref-delta' +] + +function readTypedVarInt(reader, cb) { + var type, value, shift + reader.read(1, function (end, buf) { + if (ended = end) return cb(end) + var firstByte = buf[0] + type = objectTypes[(firstByte >> 4) & 7] + value = firstByte & 15 + shift = 4 + checkByte(firstByte) + }) + + function checkByte(byte) { + if (byte & 0x80) + reader.read(1, gotByte) + else + cb(null, type, value) + } + + function gotByte(end, buf) { + if (ended = end) return cb(end) + var byte = buf[0] + value += (byte & 0x7f) << shift + shift += 7 + checkByte(byte) + } +} + +function readVarInt(reader, cb) { + var value = 0, shift = 0 + reader.read(1, function gotByte(end, buf) { + if (ended = end) return cb(end) + var byte = buf[0] + value += (byte & 0x7f) << shift + shift += 7 + if (byte & 0x80) + reader.read(1, gotByte) + else + cb(null, value) + }) +} + +function inflate(read) { + return toPull(zlib.createInflate())(read) +} + +Git.prototype.decodeObject = function (opts) { + var self = this + var packLink = opts.packLink + return function (read) { + var reader = Reader() + reader(read) + return u.readNext(function (cb) { + readTypedVarInt(reader, function (end, type, length) { + if (end === true) cb(new Error('Missing object type')) + else if (end) cb(end) + else if (type === 'ref-delta') getObjectFromRefDelta(length, cb) + else if (opts.type && type !== opts.type) + cb(new Error('expected type \'' + opts.type + '\' ' + + 'but found \'' + type + '\'')) + else if (opts.length && length !== opts.length) + cb(new Error('expected length ' + opts.length + ' ' + + 'but found ' + length)) + else cb(null, inflate(reader.read())) + }) + }) + + function getObjectFromRefDelta(length, cb) { + console.error('read from ref delta') + reader.read(20, function (end, sourceHash) { + if (end) return cb(end) + var inflatedReader = Reader() + pull(reader.read(), inflate, inflatedReader) + readVarInt(inflatedReader, function (err, expectedSourceLength) { + if (err) return cb(err) + readVarInt(inflatedReader, function (err, expectedTargetLength) { + if (err) return cb(err) + // console.error('getting object', sourceHash) + var offset = opts.idx.find(sourceHash) + if (!offset) return cb(null, 'missing source object ' + + sourcehash.toString('hex')) + console.error('get pack', opts.packLink, offset.offset, offset.next) + var readSource = pull( + self.app.readBlob(opts.packLink, { + start: offset.offset, + end: offset.next + }), + self.decodeObject({ + type: opts.type, + length: expectedSourceLength, + packLink: opts.packLink, + idx: opts.idx + }) + ) + // console.error('patching', length, expectedTargetLength) + cb(null, patchObject(inflatedReader, length, readSource, expectedTargetLength)) + }) + }) + }) + } + } +} + +function readOffsetSize(cmd, reader, readCb) { + var offset = 0, size = 0 + + function addByte(bit, outPos, cb) { + if (cmd & (1 << bit)) + reader.read(1, function (err, buf) { + if (err) readCb(err) + else cb(buf[0] << (outPos << 3)) + }) + else + cb(0) + } + + addByte(0, 0, function (val) { + offset = val + addByte(1, 1, function (val) { + offset |= val + addByte(2, 2, function (val) { + offset |= val + addByte(3, 3, function (val) { + offset |= val + addSize() + }) + }) + }) + }) + function addSize() { + addByte(4, 0, function (val) { + size = val + addByte(5, 1, function (val) { + size |= val + addByte(6, 2, function (val) { + size |= val + readCb(null, offset, size || 0x10000) + }) + }) + }) + } +} + +function patchObject(deltaReader, deltaLength, readSource, targetLength) { + var srcBuf + var ended + // console.error('patching', deltaLength, targetLength) + + return u.readNext(function (cb) { + pull(readSource, u.pullConcat(function (err, buf) { + if (err) return cb(err) + srcBuf = buf + cb(null, read) + })) + }) + + function read(abort, cb) { + // console.error('pa', abort, ended) + if (ended) return cb(ended) + deltaReader.read(1, function (end, dBuf) { + // console.error("read", end, dBuf) + // if (ended = end) return console.error('patched', deltaLength, targetLength, end), cb(end) + if (ended = end) return cb(end) + var cmd = dBuf[0] + // console.error('cmd', cmd & 0x80, cmd) + if (cmd & 0x80) + // skip a variable amount and then pass through a variable amount + readOffsetSize(cmd, deltaReader, function (err, offset, size) { + // console.error('offset', err, offset, size) + if (err) return earlyEnd(err) + var buf = srcBuf.slice(offset, offset + size) + // console.error('buf', buf) + cb(end, buf) + }) + else if (cmd) + // insert `cmd` bytes from delta + deltaReader.read(cmd, cb) + else + cb(new Error("unexpected delta opcode 0")) + }) + + function earlyEnd(err) { + cb(err === true ? new Error('stream ended early') : err) + } + } +} + +Git.prototype.getCommit = function (opts, cb) { + this.getObject(opts, function (err, buf) { + if (err) return cb(err) + var commit = { + body: buf.toString('ascii') + } + cb(null, commit) + }) +} diff --git a/lib/render-msg.js b/lib/render-msg.js index c0001fc..ef23b3f 100644 --- a/lib/render-msg.js +++ b/lib/render-msg.js @@ -25,11 +25,7 @@ RenderMsg.prototype.toUrl = function (href) { } RenderMsg.prototype.linkify = function (text) { - var arr = text.split(u.ssbRefRegex) - for (var i = 1; i < arr.length; i += 2) { - arr[i] = h('a', {href: this.toUrl(arr[i])}, arr[i]) - } - return arr + return this.render.linkify(text) } function token() { @@ -482,14 +478,20 @@ RenderMsg.prototype.gitUpdate = function (cb) { !isNaN(size) ? [self.render.formatSize(size), ' '] : '', self.c.refs ? h('ul', Object.keys(self.c.refs).map(function (ref) { var id = self.c.refs[ref] - return h('li', - ref.replace(/^refs\/(heads|tags)\//, ''), ': ', - id ? h('code', id) : h('em', 'deleted')) + var type = /^refs\/tags/.test(ref) ? 'tag' : 'commit' + var path = id && ('/git/' + type + '/' + encodeURIComponent(id) + + '?head=' + encodeURIComponent(self.msg.key)) + return h('li', + ref.replace(/^refs\/(heads|tags)\//, ''), ': ', + id ? h('a', {href: self.render.toUrl(path)}, h('code', id)) + : h('em', 'deleted')) })) : '', Array.isArray(self.c.commits) ? h('ul', self.c.commits.map(function (commit) { - return h('li', - h('code', String(commit.sha1).substr(0, 8)), ' ', + var path = '/git/commit/' + encodeURIComponent(commit.sha1) + + '?head=' + encodeURIComponent(self.msg.key) + return h('li', h('a', {href: self.render.toUrl(path)}, + h('code', String(commit.sha1).substr(0, 8))), ' ', self.linkify(String(commit.title)), self.gitCommitBody(commit.body) ) diff --git a/lib/render.js b/lib/render.js index dd6d55e..02d7285 100644 --- a/lib/render.js +++ b/lib/render.js @@ -148,6 +148,14 @@ Render.prototype.formatSize = function (size) { return size.toFixed(2) + ' MB' } +Render.prototype.linkify = function (text) { + var arr = text.split(u.ssbRefRegex) + for (var i = 1; i < arr.length; i += 2) { + arr[i] = h('a', {href: this.toUrl(arr[i])}, arr[i]) + } + return arr +} + Render.prototype.toUrl = function (href) { if (!href) return href var mentions = this._mentions diff --git a/lib/serve.js b/lib/serve.js index 8d3eeba..4e8c15d 100644 --- a/lib/serve.js +++ b/lib/serve.js @@ -119,6 +119,7 @@ Serve.prototype.go = function () { else if (data.action === 'publish') self.publishJSON(next) else if (data.action === 'vote') self.publishVote(next) else if (data.action === 'contact') self.publishContact(next) + else if (data.action === 'want-blobs') self.wantBlobs(next) else next() } @@ -172,6 +173,22 @@ Serve.prototype.publishContact = function (cb) { this.publish(content, cb) } +Serve.prototype.wantBlobs = function (cb) { + var self = this + if (!self.data.blob_ids) return cb() + var ids = self.data.blob_ids.split(',') + if (!ids.every(u.isRef)) return cb(new Error('bad blob ids ' + ids.join(','))) + var done = multicb({pluck: 1}) + ids.forEach(function (id) { + self.app.sbot.blobs.want(id, done()) + }) + done(function (err) { + if (err) return cb(err) + // self.note = h('div', 'wanted blobs: ' + ids.join(', ') + '.') + cb() + }) +} + Serve.prototype.publish = function (content, cb) { var self = this var done = multicb({pluck: 1, spread: true}) @@ -257,6 +274,7 @@ Serve.prototype.path = function (url) { case '/emoji': return this.emoji(m[2]) case '/contacts': return this.contacts(m[2]) case '/about': return this.about(m[2]) + case '/git': return this.git(m[2]) } return this.respond(404, 'Not found') } @@ -1063,6 +1081,20 @@ function catchHTMLError() { } } +function catchTextError() { + return function (read) { + var ended + return function (abort, cb) { + if (ended) return cb(ended) + read(abort, function (end, data) { + if (!end || end === true) return cb(end, data) + ended = true + cb(null, end.stack + '\n') + }) + } + } +} + function styles() { return fs.readFileSync(path.join(__dirname, '../static/styles.css'), 'utf8') } @@ -1117,6 +1149,7 @@ Serve.prototype.wrapPage = function (title, searchQ) { 'published ', self.app.render.msgLink(self.publishedMsg, done()) ) : '', + // self.note, content ))) done(cb) @@ -1245,6 +1278,65 @@ Serve.prototype.wrapUserFeed = function (isScrolled, id) { }) } +Serve.prototype.git = function (url) { + var m = /^\/?([^\/]*)\/?(.*)?$/.exec(url) + switch (m[1]) { + case 'commit': return this.gitCommit(m[2]) + case 'tag': return this.gitCommit(m[2]) + case 'raw': return this.gitRaw(m[2]) + default: return this.respond(404, 'Not found') + } +} + +Serve.prototype.gitRaw = function (rev) { + var self = this + if (!/[0-9a-f]{24}/.test(rev)) { + return pull( + pull.once('\'' + rev + '\' is not a git object id'), + self.respondSink(400, {'Content-Type': 'text/plain'}) + ) + } + + var headMsgId = self.query.head + pull( + self.app.git.readObject({ + id: rev, + headMsgId: headMsgId, + }), + catchTextError(), + self.respondSink(200, {'Content-type': 'text/plain'}) + ) +} + +Serve.prototype.gitCommit = function (rev) { + var self = this + if (!/[0-9a-f]{24}/.test(rev)) { + return pull( + ph('div.error', 'rev is not a git object id'), + self.wrapPage('git'), + self.respondSink(400) + ) + } + + self.app.git.getCommit({ + id: rev, + headMsgId: self.query.head, + }, function (err, commit) { + if (err && err.name === 'BlobNotFoundError') + return self.askWantBlobs(err.links) + if (err) return pull( + pull.once(u.renderError(err).outerHTML), + self.wrapPage('git object ' + rev), + self.respondSink(400) + ) + pull( + pull.once(h('pre', self.app.render.linkify(commit.body)).outerHTML), + self.wrapPage('git object ' + rev), + self.respondSink(200) + ) + }) +} + Serve.prototype.wrapPublic = function (opts) { var self = this return u.hyperwrap(function (thread, cb) { @@ -1260,6 +1352,31 @@ Serve.prototype.wrapPublic = function (opts) { }) } +Serve.prototype.askWantBlobs = function (links) { + var self = this + pull( + ph('form', {action: '', method: 'post'}, [ + ph('section', [ + ph('h3', 'Missing blobs'), + ph('p', 'The application needs these blobs to continue:'), + ph('table', links.map(u.toLink).map(function (link) { + if (!u.isRef(link.link)) return + return ph('tr', [ + ph('td', ph('code', link.link)), + ph('td', self.app.render.formatSize(link.size)), + ]) + })), + ph('input', {type: 'hidden', name: 'action', value: 'want-blobs'}), + ph('input', {type: 'hidden', name: 'blob_ids', + value: links.map(u.linkDest).join(',')}), + ph('p', ph('input', {type: 'submit', value: 'Want Blobs'})) + ]) + ]), + self.wrapPage('missing blobs'), + self.respondSink(409) + ) +} + Serve.prototype.wrapPrivate = function (opts) { var self = this return u.hyperwrap(function (thread, cb) { diff --git a/lib/util.js b/lib/util.js index 5546716..dbefa14 100644 --- a/lib/util.js +++ b/lib/util.js @@ -62,6 +62,10 @@ u.hyperwrap = function (fn) { } } +u.toLink = function (link) { + return typeof link === 'string' ? {link: link} : link +} + u.linkDest = function (link) { return typeof link === 'string' ? link : link && link.link || link } @@ -114,3 +118,19 @@ u.isMsgEncrypted = function (msg) { var c = msg && msg.value.content return typeof c === 'string' } + +u.pullConcat = function (cb) { + return pull.collect(function (err, bufs) { + if (err) return cb(err) + cb(null, Buffer.concat(bufs)) + }) +} + +u.customError = function (name) { + return function (message) { + var error = new Error(message) + error.name = name + error.stack = error.stack.replace(/^ at .*\n/m, '') + return error + } +} diff --git a/package.json b/package.json index 7b77ca5..2a78964 100644 --- a/package.json +++ b/package.json @@ -13,11 +13,13 @@ "mime-types": "^2.1.12", "multicb": "^1.2.1", "pull-cat": "^1.1.11", + "pull-git-packidx-parser": "^1.0.0", "pull-hash": "^1.0.0", "pull-hyperscript": "^0.2.2", "pull-identify-filetype": "^1.1.0", "pull-paginate": "^1.0.0", "pull-paramap": "^1.2.1", + "pull-reader": "^1.2.9", "pull-stream": "^3.5.0", "ssb-contact": "^1.0.0", "ssb-marked": "^0.7.1", -- cgit v1.2.3 From b66bcecec258b0a2631ec338501afa9409882fe8 Mon Sep 17 00:00:00 2001 From: cel Date: Sun, 28 May 2017 18:42:44 -1000 Subject: Slice packfiles manually Support vanilla multiblob --- lib/app.js | 19 +++++++++++++------ lib/git.js | 4 ++-- lib/util.js | 28 ++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 8 deletions(-) (limited to 'lib/app.js') diff --git a/lib/app.js b/lib/app.js index 1daf064..8ef859f 100644 --- a/lib/app.js +++ b/lib/app.js @@ -177,10 +177,7 @@ App.prototype.addBlob = function (cb) { done(function (err, hash, add) { cb(err, hash) }) - return pull( - hasher(hashCb), - this.sbot.blobs.add(addCb) - ) + return sink } App.prototype.pushBlob = function (id, cb) { @@ -188,15 +185,25 @@ App.prototype.pushBlob = function (id, cb) { this.sbot.blobs.push(id, cb) } -App.prototype.readBlob = function (link, opts) { +App.prototype.readBlob = function (link) { link = u.toLink(link) - opts = opts || {} return this.sbot.blobs.get({ hash: link.link, size: link.size, + }) +} + +App.prototype.readBlobSlice = function (link, opts) { + if (this.sbot.blobs.getSlice) return this.sbot.blobs.getSlice({ + hash: link.link, + size: link.size, start: opts.start, end: opts.end, }) + return pull( + this.readBlob(link), + u.pullSlice(opts.start, opts.end) + ) } App.prototype.ensureHasBlobs = function (links, cb) { diff --git a/lib/git.js b/lib/git.js index 07061e2..1360a6f 100644 --- a/lib/git.js +++ b/lib/git.js @@ -82,7 +82,7 @@ Git.prototype.openObject = function (opts, cb) { Git.prototype.readObject = function (obj) { return pull( - this.app.readBlob(obj.packLink, {start: obj.offset, end: obj.next}), + this.app.readBlobSlice(obj.packLink, {start: obj.offset, end: obj.next}), this.decodeObject({ type: obj.type, length: obj.length, @@ -310,7 +310,7 @@ Git.prototype.decodeObject = function (opts) { if (!offset) return cb(null, 'missing source object ' + sourcehash.toString('hex')) var readSource = pull( - self.app.readBlob(opts.packLink, { + self.app.readBlobSlice(opts.packLink, { start: offset.offset, end: offset.next }), diff --git a/lib/util.js b/lib/util.js index e8fc435..d25ecaf 100644 --- a/lib/util.js +++ b/lib/util.js @@ -141,3 +141,31 @@ u.escapeHTML = function (html) { .replace(//g, '>') } + +u.pullSlice = function (start, end) { + if (end == null) end = Infinity + var offset = 0 + return function (read) { + return function (abort, cb) { + if (abort) read(abort, cb) + else if (offset >= end) read(true, function (err) { + cb(err || true) + }) + else if (offset < start) read(null, function next(err, data) { + if (err) return cb(err) + offset += data.length + if (offset <= start) read(null, next) + else if (offset < end) cb(null, + data.slice(data.length - (offset - start))) + else cb(null, data.slice(data.length - (offset - start), + data.length - (offset - end))) + }) + else read(null, function (err, data) { + if (err) return cb(err) + offset += data.length + if (offset <= end) cb(null, data) + else cb(null, data.slice(0, data.length - (offset - end))) + }) + } + } +} -- cgit v1.2.3