Attempt to fix pagination issues

- Filter messages before passing them to paginate function,
  so that the paginate function picks the correct one as the last one.
- Remove obsolete lt/gt hack
- Try to handle sortByTimestamp for /mentions

TODO: avoid calling filterMsg multiple times per message

2 files changed, 41 insertions, 24 deletions

diff --git a/lib/app.js b/lib/app.js
index fbdbb8f..b554f31 100644
--- a/lib/app.js
+++ b/lib/app.js
@@ -390,15 +390,13 @@ App.prototype.createLogStream = function (opts) {
 
 App.prototype.createFeedStream = function (opts) {
   // work around opts.gt being treated as opts.gte sometimes
-  var limit = Number(opts.limit)
-  if (opts.gt && limit && !opts.reverse) return pull(
-    this.sbot.createFeedStream(u.mergeOpts(opts, {limit: opts.limit + 1})),
+  return pull(
+    this.sbot.createFeedStream(opts),
     pull.filter(function (msg) {
-      return msg && msg.value.timestamp !== opts.gt
-    }),
-    limit && pull.take(limit)
+      var ts = msg && msg.value && msg.value.timestamp
+      return typeof ts === 'number' && ts !== opts.gt && ts !== opts.lt
+    })
   )
-  return this.sbot.createFeedStream(opts)
 }
 
 var stateVals = {
@@ -593,25 +591,38 @@ App.prototype.filter = function (plugin, opts, filter) {
   if (plugin === this.sbot.backlinks) {
     var c = filter && filter.value && filter.value.content
     var filteringByType = c && c.type
-    if (!filteringByType) index = 'DTS'
+    if (opts.sortByTimestamp) index = 'DTA'
+    else if (filteringByType) index = 'DTS'
   }
-  // work around flumeview-query not supporting $lt/$gt.
-  // %FCIv0D7JQyERznC18p8Dc1KtN6SLeJAl1sR5DAIr/Ek=.sha256
+  var filterOpts = {
+    $gt: opts.gt,
+    $lt: opts.lt,
+  }
+  return plugin.read({
+    index: index,
+    reverse: opts.reverse,
+    limit: limit || undefined,
+    query: [{$filter: u.mergeOpts(filter, opts.sortByTimestamp ? {
+      value: {
+        timestamp: filterOpts
+      }
+    } : {
+      timestamp: filterOpts
+    })}]
+  })
+}
+
+App.prototype.filterMessages = function (opts) {
+  var self = this
+  var limit = Number(opts.limit)
   return pull(
-    plugin.read({
-      index: index,
-      reverse: opts.reverse,
-      limit: limit ? (limit + 1) : undefined,
-      query: [{$filter: u.mergeOpts(filter, {
-        timestamp: {
-          $gte: opts.gt,
-          $lte: opts.lt,
-        }
-      })}]
-    }),
-    pull.filter(function (msg) {
-      return msg && msg.timestamp !== opts.lt && msg.timestamp !== opts.gt
-    }),
+    paramap(function (msg, cb) {
+      self.filterMsg(msg, opts, function (err, show) {
+        if (err) return cb(err)
+        cb(null, show ? msg : null)
+      })
+    }, 4),
+    pull.filter(Boolean),
     limit && pull.take(limit)
   )
 }
diff --git a/lib/serve.js b/lib/serve.js
index 9b4dcd4..e9b8f2b 100644
--- a/lib/serve.js
+++ b/lib/serve.js
@@ -1343,6 +1343,12 @@ Serve.prototype.renderThreadPaginated = function (opts, feedId, q) {
   }
 
   return pull(
+    self.app.filterMessages({
+      feed: opts && opts.feed,
+      msgId: opts && opts.msgId,
+      filter: this.query.filter,
+      limit: Number(this.query.limit) || 12,
+    }),
     paginate(
       function onFirst(msg, cb) {
         var num = feedId ? msg.value.sequence :