rebases need to track changed indices

src/io/lacuna/bifurcan/DurableList.java

@@ -10,7 +10,7 @@
 
 import static io.lacuna.bifurcan.durable.codecs.Core.decodeBlock;
 
-public class DurableList<V> extends IList.Mixin<V> implements IDurableCollection {
+public class DurableList<V> extends IList.Mixin<V> implements IList.Durable<V> {
 
   private final DurableInput.Pool bytes;
   private final Root root;

src/io/lacuna/bifurcan/IDiffMap.java

@@ -80,63 +80,4 @@ default Iterator<IEntry<K, V>> iterator() {
         Util.skipIndices(underlying().entries().iterator(), removedIndices().iterator()),
         added().entries().iterator());
   }
-
-  static PrimitiveIterator.OfLong mergedRemovedIndices(IList<IDiffMap<?, ?>> diffStack) {
-    assert diffStack.stream().allMatch(m -> m instanceof IMap.Durable);
-
-    // isolate the removed indices which only apply to the underlying collection (which is potentially shrinking with
-    // each new stacked diff)
-    IList<Iterator<Long>> iterators = new LinearList<>();
-    long underlyingSize = diffStack.first().underlying().size();
-    long removed = 0;
-    for (IDiffMap<?, ?> m : diffStack) {
-      long remainingUnderlyingSize = underlyingSize - removed;
-      ISortedSet<Long> s = m.removedIndices().slice(0L, remainingUnderlyingSize - 1);
-      iterators.addLast(s.iterator());
-      removed += s.size();
-    }
-
-    return Util.mergedRemovedIndices(iterators);
-  }
-
-  static <K, V> Iterator<IEntry.WithHash<K, V>> mergedAddedEntries(IList<IDiffMap<K, V>> diffStack) {
-    assert diffStack.stream().allMatch(m -> m instanceof IMap.Durable);
-
-    // isolate the removed indices which only apply to the added entries
-    IList<Iterator<Long>> iterators = new LinearList<>();
-    long underlyingSize = diffStack.first().underlying().size();
-    long removed = 0;
-    for (IDiffMap<K, V> m : diffStack) {
-      long remainingUnderlyingSize = underlyingSize - removed;
-      ISortedSet<Long> underlyingIndices = m.removedIndices().slice(0L, remainingUnderlyingSize - 1);
-      ISortedSet<Long> addedIndices = m.removedIndices().slice(remainingUnderlyingSize, Long.MAX_VALUE);
-      iterators.addLast(Iterators.map(addedIndices.iterator(), n -> n - remainingUnderlyingSize));
-      removed += underlyingIndices.size();
-    }
-
-    SkipTable.Writer writer = new SkipTable.Writer();
-    Util.mergedRemovedIndices(iterators).forEachRemaining((long idx) -> writer.append(idx, 0));
-
-    // for this to consume too much memory would require >100M entries being repeatedly overwritten within the stack
-    // of diffs, which implies that many entries being in-memory at once, which seems far-fetched enough that I'm not
-    // going to worry about it for now
-    // TODO: worry about it
-    ISortedSet<Long> removedIndices = writer.toOffHeapMap().keys();
-
-    // get the hash-sorted entries (which are in the same order as entries() because it's a durable map) from each
-    // added() and filter out the removed entries from each
-    IList<Iterator<IEntry.WithHash<K, V>>> sortedEntries = new LinearList<>();
-    long offset = 0;
-    for (IDiffMap<K, V> m : diffStack) {
-      long size = m.added().size();
-      long currOffset = offset;
-      sortedEntries.addLast(
-          Util.skipIndices(
-              m.added().hashSortedEntries(),
-              Iterators.map(removedIndices.slice(currOffset, currOffset + size - 1).iterator(), n -> n - currOffset)));
-      offset += size;
-    }
-
-    return Iterators.mergeSort(sortedEntries, Comparator.comparing(IEntry.WithHash::keyHash));
-  }
 }

src/io/lacuna/bifurcan/IDurableCollection.java

@@ -72,6 +72,14 @@ default IDurableCollection decode(IDurableEncoding encoding) {
   Root root();
 
   interface Rebase {
+    Fingerprint original();
+
+    Fingerprint updated();
+
+    ISortedMap<Long, Long> updatedIndices();
+
+    Root root();
+
     <T extends IDurableCollection> T apply(T collection);
   }
 
@@ -84,27 +92,15 @@ default Rebase compact(ISet<Fingerprint> compactSet) {
       throw new IllegalArgumentException("unexpected roots in `compactSet`: " + unexpectedRoots);
     }
 
-    System.out.println(compactSet + " " + compactGraph + " " + root().dependencyGraph());
     ISet<Fingerprint> reachable = Set.from(Graphs.bfsVertices(fingerprint, compactGraph::out));
     if (reachable.size() < compactSet.size()) {
       throw new IllegalArgumentException("disconnected elements in `compactSet`: " + compactSet.difference(reachable));
     }
 
-    Fingerprint compacted = Core.compacting(
-        compactSet,
-        () -> FileOutput.write(
-            root().path().getParent(),
-            Map.empty(),
-            acc -> Core.encodeSingleton(this, encoding(), acc)));
-
-    IMap<Fingerprint, Fingerprint> rebases = new Map<Fingerprint, Fingerprint>().put(fingerprint, compacted);
-    return new Rebase() {
-      @Override
-      public <T extends IDurableCollection> T apply(T c) {
-        Path dir = c.root().path().getParent();
-        Fingerprint f = FileOutput.write(dir, rebases, acc -> Core.encodeSingleton(c, c.encoding(), acc));
-        return (T) Roots.open(dir, f).decode(c.encoding());
-      }
-    };
+    if (compactSet.size() < 2) {
+      throw new IllegalArgumentException("there must be at least two elements in `compactSet`");
+    }
+
+    return Core.compact(compactSet, this);
   }
 }

src/io/lacuna/bifurcan/IList.java

@@ -52,6 +52,10 @@ public IList<V> clone() {
     }
   }
 
+  interface Durable<V> extends IList<V>, IDurableCollection {
+    IDurableEncoding.List encoding();
+  }
+
   default IList<V> update(long idx, Function<V, V> updateFn) {
     return set(idx, updateFn.apply(nth(idx)));
   }

src/io/lacuna/bifurcan/ISet.java

@@ -50,6 +50,10 @@ public ISet<V> clone() {
     }
   }
 
+  interface Durable<V> extends ISet<V>, IDurableCollection {
+    IDurableEncoding.Set encoding();
+  }
+
   /**
    * @return the hash function used by the set
    */

src/io/lacuna/bifurcan/diffs/DiffList.java

@@ -48,7 +48,6 @@ public IList<V> suffix() {
 
   @Override
   public IList<V> slice(long start, long end) {
-
     long pSize = prefix().size();
     List<V> prefix = start < pSize
         ? this.prefix.slice(start, Math.min(pSize, end))
@@ -98,6 +97,10 @@ public IList<V> addFirst(V value) {
 
   @Override
   public IList<V> removeLast() {
+    if (isLinear()) {
+      super.hash = -1;
+    }
+
     if (suffix.size() > 0) {
       List<V> suffixPrime = suffix.removeLast();
       return isLinear() ? this : new DiffList<>(underlying, prefix, suffixPrime, slice);
@@ -117,6 +120,10 @@ public IList<V> removeLast() {
 
   @Override
   public IList<V> removeFirst() {
+    if (isLinear()) {
+      super.hash = -1;
+    }
+
     if (prefix.size() > 0) {
       List<V> prefixPrime = prefix.removeFirst();
       return isLinear() ? this : new DiffList<>(underlying, prefixPrime, suffix, slice);
@@ -136,6 +143,10 @@ public IList<V> removeFirst() {
 
   @Override
   public IList<V> set(long idx, V value) {
+    if (isLinear()){
+      super.hash = -1;
+    }
+
     if (idx < prefix.size()) {
       List<V> prefixPrime = prefix.set(idx, value);
       return isLinear() ? this : new DiffList<>(underlying, prefixPrime, suffix, slice);

src/io/lacuna/bifurcan/diffs/DiffMap.java

@@ -33,9 +33,19 @@ public DiffMap<K, V> put(K key, V value, BinaryOperator<V> merge) {
 
     if (addedPrime.size() != addedSize) {
       ISortedSet<Long> removedIndicesPrime = idx.isPresent() ? removedIndices.add(idx.getAsLong()) : removedIndices;
-      return isLinear() ? this : new DiffMap<>(underlying, addedPrime, removedIndicesPrime);
+      if (isLinear()) {
+        super.hash = -1;
+        return this;
+      } else {
+        return new DiffMap<>(underlying, addedPrime, removedIndicesPrime);
+      }
     } else {
-      return isLinear() ? this : new DiffMap<>(underlying, addedPrime, removedIndices);
+      if (isLinear()) {
+        super.hash = -1;
+        return this;
+      } else {
+        return new DiffMap<>(underlying, addedPrime, removedIndices);
+      }
     }
   }
 
@@ -44,7 +54,12 @@ public DiffMap<K, V> remove(K key) {
     IMap<K, V> addedPrime = added.remove(key);
     OptionalLong idx = underlying.indexOf(key);
     ISortedSet<Long> removedIndicesPrime = idx.isPresent() ? removedIndices.add(idx.getAsLong()) : removedIndices;
-    return isLinear() ? this : new DiffMap<>(underlying, addedPrime, removedIndicesPrime);
+    if (isLinear()) {
+      super.hash = -1;
+      return this;
+    } else {
+      return new DiffMap<>(underlying, addedPrime, removedIndicesPrime);
+    }
   }
 
   @Override