From 5da694977fea54846d20466bbed8bd48b5e346ec Mon Sep 17 00:00:00 2001 From: ztellman Date: Fri, 21 Aug 2020 19:15:15 -0700 Subject: [PATCH] some test failures, but getting closer --- src/io/lacuna/bifurcan/DurableEncodings.java | 4 +- src/io/lacuna/bifurcan/DurableInput.java | 2 +- src/io/lacuna/bifurcan/DurableList.java | 16 +- src/io/lacuna/bifurcan/DurableMap.java | 31 ++-- src/io/lacuna/bifurcan/ICollection.java | 19 +-- src/io/lacuna/bifurcan/IDiffMap.java | 68 ++++++--- .../lacuna/bifurcan/IDurableCollection.java | 54 +++++-- src/io/lacuna/bifurcan/IDurableEncoding.java | 2 +- src/io/lacuna/bifurcan/IMap.java | 6 +- src/io/lacuna/bifurcan/ISortedSet.java | 2 +- .../lacuna/bifurcan/durable/Fingerprints.java | 25 ++- src/io/lacuna/bifurcan/durable/Rebases.java | 9 +- src/io/lacuna/bifurcan/durable/Roots.java | 14 +- .../durable/codecs/{Util.java => Core.java} | 143 +++++++++++++----- .../codecs/{Diffs.java => DiffHashMap.java} | 43 +++++- .../bifurcan/durable/codecs/HashMap.java | 2 + .../durable/codecs/HashMapEntries.java | 4 +- .../lacuna/bifurcan/durable/codecs/List.java | 2 +- .../bifurcan/durable/codecs/Reference.java | 14 +- .../bifurcan/durable/codecs/SkipTable.java | 6 + .../bifurcan/durable/codecs/TempStream.java | 4 +- .../bifurcan/durable/io/DurableBuffer.java | 9 ++ .../bifurcan/durable/io/FileOutput.java | 44 +++--- test/bifurcan/durable_test.clj | 94 ++++++------ 24 files changed, 400 insertions(+), 217 deletions(-) rename src/io/lacuna/bifurcan/durable/codecs/{Util.java => Core.java} (53%) rename src/io/lacuna/bifurcan/durable/codecs/{Diffs.java => DiffHashMap.java} (60%) diff --git a/src/io/lacuna/bifurcan/DurableEncodings.java b/src/io/lacuna/bifurcan/DurableEncodings.java index f9a29e2..f04e2d8 100644 --- a/src/io/lacuna/bifurcan/DurableEncodings.java +++ b/src/io/lacuna/bifurcan/DurableEncodings.java @@ -9,8 +9,8 @@ import java.util.Objects; import java.util.function.*; -import static io.lacuna.bifurcan.durable.codecs.Util.decodeBlock; -import static io.lacuna.bifurcan.durable.codecs.Util.encodeBlock; +import static io.lacuna.bifurcan.durable.codecs.Core.decodeBlock; +import static io.lacuna.bifurcan.durable.codecs.Core.encodeBlock; /** * Utility methods for constructing {@link IDurableEncoding}s. diff --git a/src/io/lacuna/bifurcan/DurableInput.java b/src/io/lacuna/bifurcan/DurableInput.java index 90c663c..099935a 100644 --- a/src/io/lacuna/bifurcan/DurableInput.java +++ b/src/io/lacuna/bifurcan/DurableInput.java @@ -137,7 +137,7 @@ default String hexBytes() { } /** - * @returnb the bounds for this input + * @return the bounds for this input */ Bounds bounds(); diff --git a/src/io/lacuna/bifurcan/DurableList.java b/src/io/lacuna/bifurcan/DurableList.java index 0496225..722e1d8 100644 --- a/src/io/lacuna/bifurcan/DurableList.java +++ b/src/io/lacuna/bifurcan/DurableList.java @@ -3,7 +3,6 @@ import io.lacuna.bifurcan.durable.Dependencies; import io.lacuna.bifurcan.durable.Roots; import io.lacuna.bifurcan.durable.codecs.List; -import io.lacuna.bifurcan.durable.codecs.SkipTable; import io.lacuna.bifurcan.durable.io.DurableBuffer; import io.lacuna.bifurcan.durable.io.FileOutput; import io.lacuna.bifurcan.utils.Iterators; @@ -11,7 +10,7 @@ import java.nio.file.Path; import java.util.Iterator; -import static io.lacuna.bifurcan.durable.codecs.Util.decodeBlock; +import static io.lacuna.bifurcan.durable.codecs.Core.decodeBlock; public class DurableList implements IDurableCollection, IList { @@ -38,17 +37,8 @@ public static DurableList open(IDurableEncoding.List encoding, Path path) } public static DurableList from(Iterator elements, IDurableEncoding.List encoding, Path directory) { - Dependencies.enter(); - DurableBuffer acc = new DurableBuffer(); - List.encode(elements, encoding, acc); - - FileOutput file = new FileOutput(Dependencies.exit(), Map.empty()); - DurableOutput out = DurableOutput.from(file); - acc.flushTo(out); - out.close(); - - Path path = file.moveTo(directory); - return (DurableList) Roots.open(path).decode(encoding); + Fingerprint f = FileOutput.write(directory, Map.empty(), acc -> List.encode(elements, encoding, acc)); + return (DurableList) Roots.open(directory, f).decode(encoding); } @Override diff --git a/src/io/lacuna/bifurcan/DurableMap.java b/src/io/lacuna/bifurcan/DurableMap.java index 2f7e6e3..25d7789 100644 --- a/src/io/lacuna/bifurcan/DurableMap.java +++ b/src/io/lacuna/bifurcan/DurableMap.java @@ -45,26 +45,12 @@ public static DurableMap open(Path path, IDurableEncoding.Map encod return (DurableMap) Roots.open(path).decode(encoding); } - public static void encode(Iterator> entries, IDurableEncoding.Map encoding, int maxRealizedEntries, DurableOutput out) { - HashMap.encodeSortedEntries(HashMap.sortEntries(entries, encoding, maxRealizedEntries), encoding, out); - } - - public static DurableMap decode(IDurableEncoding.Map encoding, Root root, DurableInput.Pool pool ) { - return HashMap.decode(encoding, root, pool); - } - public static DurableMap from(Iterator> entries, IDurableEncoding.Map encoding, Path directory, int maxRealizedEntries) { - Dependencies.enter(); - DurableBuffer acc = new DurableBuffer(); - encode(entries, encoding, maxRealizedEntries, acc); - - FileOutput file = new FileOutput(Dependencies.exit(), Map.empty()); - DurableOutput out = DurableOutput.from(file); - acc.flushTo(out); - out.close(); - - Path path = file.moveTo(directory); - return (DurableMap) Roots.open(path).decode(encoding); + Fingerprint f = FileOutput.write( + directory, + Map.empty(), + acc -> HashMap.encodeSortedEntries(HashMap.sortEntries(entries, encoding, maxRealizedEntries), encoding, acc)); + return (DurableMap) Roots.open(directory, f).decode(encoding); } private Iterator chunkedEntries(long offset) { @@ -136,11 +122,16 @@ public IEntry.WithHash nth(long idx) { @Override public Iterator> iterator() { + return (Iterator) hashSortedEntries(); + } + + @Override + public Iterator> hashSortedEntries() { return Iterators.flatMap( chunkedEntries(0), chunk -> Iterators.map( chunk.entries(0), - e -> IEntry.of((K) e.key(), (V) e.value()))); + e -> IEntry.of(e.keyHash(), (K) e.key(), (V) e.value()))); } @Override diff --git a/src/io/lacuna/bifurcan/ICollection.java b/src/io/lacuna/bifurcan/ICollection.java index c5bd6c5..f44c3f9 100644 --- a/src/io/lacuna/bifurcan/ICollection.java +++ b/src/io/lacuna/bifurcan/ICollection.java @@ -2,15 +2,12 @@ import io.lacuna.bifurcan.durable.Dependencies; import io.lacuna.bifurcan.durable.Roots; -import io.lacuna.bifurcan.durable.codecs.Diffs; -import io.lacuna.bifurcan.durable.codecs.HashMap; -import io.lacuna.bifurcan.durable.codecs.Util; +import io.lacuna.bifurcan.durable.codecs.Core; import io.lacuna.bifurcan.durable.io.DurableBuffer; import io.lacuna.bifurcan.durable.io.FileOutput; import java.nio.file.Path; import java.util.Iterator; -import java.util.Optional; /** * @author ztellman @@ -93,17 +90,7 @@ default Iterator iterator() { C clone(); default C save(IDurableEncoding encoding, Path directory) { - Dependencies.enter(); - DurableBuffer acc = new DurableBuffer(); - - Util.encodeSingleton(this, encoding, acc); - - FileOutput file = new FileOutput(Dependencies.exit(), Map.empty()); - DurableOutput out = DurableOutput.from(file); - acc.flushTo(out); - out.close(); - - Path path = file.moveTo(directory); - return (C) Roots.open(path).decode(encoding); + IDurableCollection.Fingerprint f = FileOutput.write(directory, Map.empty(), acc -> Core.encodeSingleton(this, encoding, acc)); + return (C) Roots.open(directory, f).decode(encoding); } } diff --git a/src/io/lacuna/bifurcan/IDiffMap.java b/src/io/lacuna/bifurcan/IDiffMap.java index a8ec09a..c56062e 100644 --- a/src/io/lacuna/bifurcan/IDiffMap.java +++ b/src/io/lacuna/bifurcan/IDiffMap.java @@ -1,8 +1,11 @@ package io.lacuna.bifurcan; import io.lacuna.bifurcan.diffs.Util; +import io.lacuna.bifurcan.durable.codecs.SkipTable; +import io.lacuna.bifurcan.durable.io.DurableBuffer; import io.lacuna.bifurcan.utils.Iterators; +import java.util.Comparator; import java.util.Iterator; import java.util.OptionalLong; import java.util.PrimitiveIterator; @@ -78,47 +81,62 @@ default Iterator> iterator() { added().entries().iterator()); } - static IList> diffStack(IDiffMap m) { - List> result = new List>().linear(); - IDiffMap curr = m; - for (;;) { - result.addFirst(curr); - if (curr.underlying() instanceof IDiffMap) { - curr = (IDiffMap) curr.underlying(); - } else { - break; - } - } - return result; - } + static PrimitiveIterator.OfLong mergedRemovedIndices(IList> diffStack) { + assert diffStack.stream().allMatch(m -> m instanceof IMap.Durable); - static PrimitiveIterator.OfLong compactedRemovedIndices(IList> diffStack) { + // isolate the removed indices which only apply to the underlying collection (which is potentially shrinking with + // each new stacked diff) IList> iterators = new LinearList<>(); long underlyingSize = diffStack.first().underlying().size(); long removed = 0; for (IDiffMap m : diffStack) { - long underlyingRemainingSize = underlyingSize - removed; - ISortedSet s = m.removedIndices().slice(0L, underlyingRemainingSize - 1); + long remainingUnderlyingSize = underlyingSize - removed; + ISortedSet s = m.removedIndices().slice(0L, remainingUnderlyingSize - 1); iterators.addLast(s.iterator()); removed += s.size(); } + return Util.mergedRemovedIndices(iterators); } - static Iterator> compactedAddedEntries(IList> diffStack) { + static Iterator> mergedAddedEntries(IList> diffStack) { + assert diffStack.stream().allMatch(m -> m instanceof IMap.Durable); + + // isolate the removed indices which only apply to the added entries IList> iterators = new LinearList<>(); long underlyingSize = diffStack.first().underlying().size(); long removed = 0; - for (IDiffMap m : diffStack) { - long underlyingRemainingSize = underlyingSize - removed; - ISortedSet underlyingIndices = m.removedIndices().slice(0L, underlyingRemainingSize - 1); - ISortedSet addedIndices = m.removedIndices().slice(underlyingRemainingSize, Long.MAX_VALUE); - iterators.addLast(Iterators.map(addedIndices.iterator(), n -> n - underlyingRemainingSize)); + for (IDiffMap m : diffStack) { + long remainingUnderlyingSize = underlyingSize - removed; + ISortedSet underlyingIndices = m.removedIndices().slice(0L, remainingUnderlyingSize - 1); + ISortedSet addedIndices = m.removedIndices().slice(remainingUnderlyingSize, Long.MAX_VALUE); + iterators.addLast(Iterators.map(addedIndices.iterator(), n -> n - remainingUnderlyingSize)); removed += underlyingIndices.size(); } - PrimitiveIterator.OfLong removedFromAdded = Util.mergedRemovedIndices(iterators); - IList> added = diffStack.stream().map(m -> m.added().entries()).reduce(Lists::concat).get(); - return Util.skipIndices(added.iterator(), removedFromAdded); + SkipTable.Writer writer = new SkipTable.Writer(); + Util.mergedRemovedIndices(iterators).forEachRemaining((long idx) -> writer.append(idx, 0)); + + // for this to consume too much memory would require >100M entries being repeatedly overwritten within the stack + // of diffs, which implies that many entries being in-memory at once, which seems far-fetched enough that I'm not + // going to worry about it for now + // TODO: worry about it + ISortedSet removedIndices = writer.toOffHeapMap().keys(); + + // get the hash-sorted entries (which are in the same order as entries() because it's a durable map) from each + // added() and filter out the removed entries from each + IList>> sortedEntries = new LinearList<>(); + long offset = 0; + for (IDiffMap m : diffStack) { + long size = m.added().size(); + long currOffset = offset; + sortedEntries.addLast( + Util.skipIndices( + m.added().hashSortedEntries(), + Iterators.map(removedIndices.slice(currOffset, currOffset + size - 1).iterator(), n -> n - currOffset))); + offset += size; + } + + return Iterators.mergeSort(sortedEntries, Comparator.comparing(IEntry.WithHash::keyHash)); } } diff --git a/src/io/lacuna/bifurcan/IDurableCollection.java b/src/io/lacuna/bifurcan/IDurableCollection.java index 1d67d19..047ccb7 100644 --- a/src/io/lacuna/bifurcan/IDurableCollection.java +++ b/src/io/lacuna/bifurcan/IDurableCollection.java @@ -2,21 +2,22 @@ import io.lacuna.bifurcan.durable.Bytes; import io.lacuna.bifurcan.durable.Roots; -import io.lacuna.bifurcan.durable.Util; -import io.lacuna.bifurcan.durable.io.BufferInput; -import io.lacuna.bifurcan.durable.io.DurableBuffer; +import io.lacuna.bifurcan.durable.codecs.Core; +import io.lacuna.bifurcan.durable.io.FileOutput; -import java.io.Closeable; +import java.io.File; import java.nio.ByteBuffer; import java.nio.file.Path; -import java.util.Iterator; import java.util.function.Function; -import static io.lacuna.bifurcan.durable.codecs.Util.decodeCollection; +import static io.lacuna.bifurcan.durable.codecs.Core.decodeCollection; public interface IDurableCollection { interface Fingerprint extends Comparable { + String ALGORITHM = "SHA-512"; + int HASH_BYTES = 32; + byte[] binary(); default String toHexString() { @@ -64,13 +65,46 @@ default IDurableCollection decode(IDurableEncoding encoding) { } } - interface Rebase { - T apply(T collection); - } - IDurableEncoding encoding(); DurableInput.Pool bytes(); Root root(); + + interface Rebase { + T apply(T collection); + } + + default Rebase compact(ISet compactSet) { + Fingerprint fingerprint = root().fingerprint(); + DirectedAcyclicGraph compactGraph = root().dependencyGraph().select(compactSet); + + ISet unexpectedRoots = compactGraph.top().remove(fingerprint); + if (unexpectedRoots.size() > 0) { + throw new IllegalArgumentException("unexpected roots in `compactSet`: " + unexpectedRoots); + } + + System.out.println(compactSet + " " + compactGraph + " " + root().dependencyGraph()); + ISet reachable = Set.from(Graphs.bfsVertices(fingerprint, compactGraph::out)); + if (reachable.size() < compactSet.size()) { + throw new IllegalArgumentException("disconnected elements in `compactSet`: " + compactSet.difference(reachable)); + } + + Fingerprint compacted = Core.compacting( + compactSet, + () -> FileOutput.write( + root().path().getParent(), + Map.empty(), + acc -> Core.encodeSingleton(this, encoding(), acc))); + + IMap rebases = new Map().put(fingerprint, compacted); + return new Rebase() { + @Override + public T apply(T c) { + Path dir = c.root().path().getParent(); + Fingerprint f = FileOutput.write(dir, rebases, acc -> Core.encodeSingleton(c, c.encoding(), acc)); + return (T) Roots.open(dir, f).decode(c.encoding()); + } + }; + } } diff --git a/src/io/lacuna/bifurcan/IDurableEncoding.java b/src/io/lacuna/bifurcan/IDurableEncoding.java index f6b816f..a9bded0 100644 --- a/src/io/lacuna/bifurcan/IDurableEncoding.java +++ b/src/io/lacuna/bifurcan/IDurableEncoding.java @@ -122,7 +122,7 @@ default boolean isSingleton(Object o) { interface Unityped extends Map, Set, List, Primitive { @Override default boolean isSingleton(Object o) { - return o instanceof ICollection; + return o instanceof ICollection || o instanceof java.util.Collection; } @Override diff --git a/src/io/lacuna/bifurcan/IMap.java b/src/io/lacuna/bifurcan/IMap.java index c76a403..ce0805c 100644 --- a/src/io/lacuna/bifurcan/IMap.java +++ b/src/io/lacuna/bifurcan/IMap.java @@ -1,11 +1,6 @@ package io.lacuna.bifurcan; import io.lacuna.bifurcan.diffs.DiffMap; -import io.lacuna.bifurcan.durable.Dependencies; -import io.lacuna.bifurcan.durable.Roots; -import io.lacuna.bifurcan.durable.codecs.Diffs; -import io.lacuna.bifurcan.durable.io.FileOutput; -import io.lacuna.bifurcan.durable.io.DurableBuffer; import io.lacuna.bifurcan.durable.codecs.HashMap; import io.lacuna.bifurcan.utils.Iterators; @@ -24,6 +19,7 @@ public interface IMap extends Function { interface Durable extends IMap, IDurableCollection { + IDurableEncoding.Map encoding(); } /** diff --git a/src/io/lacuna/bifurcan/ISortedSet.java b/src/io/lacuna/bifurcan/ISortedSet.java index 4c88f83..9062c20 100644 --- a/src/io/lacuna/bifurcan/ISortedSet.java +++ b/src/io/lacuna/bifurcan/ISortedSet.java @@ -90,7 +90,7 @@ default ISortedSet slice(V min, V max) { } else { long minIdx = oMinIdx.getAsLong(); long maxIdx = oMaxIdx.getAsLong(); - return Sets.from(elements().slice(minIdx, maxIdx), comparator(), v -> { + return Sets.from(elements().slice(minIdx, maxIdx + 1), comparator(), v -> { OptionalLong oIdx = floorIndex(v); if (oIdx.isPresent()) { long idx = oIdx.getAsLong(); diff --git a/src/io/lacuna/bifurcan/durable/Fingerprints.java b/src/io/lacuna/bifurcan/durable/Fingerprints.java index cddd183..5a6a2af 100644 --- a/src/io/lacuna/bifurcan/durable/Fingerprints.java +++ b/src/io/lacuna/bifurcan/durable/Fingerprints.java @@ -11,7 +11,6 @@ import java.util.Arrays; public class Fingerprints { - public static Fingerprint from(byte[] binary) { int hash = PerlHash.hash(ByteBuffer.wrap(binary)); @@ -33,17 +32,31 @@ public boolean equals(Object obj) { } return false; } + + @Override + public String toString() { + return toHexString().substring(0, 8); + } }; } - public static void encode(byte[] b, int len, DurableOutput out) { - out.writeUnsignedByte(len); - out.write(b, 0, len); + public static byte[] trim(byte[] b, int len) { + if (b.length == len) { + return b; + } else { + byte[] result = new byte[len]; + System.arraycopy(b, 0, result, 0, len); + return result; + } + } + + public static void encode(byte[] b, DurableOutput out) { + out.writeUnsignedByte(b.length); + out.write(b); } public static void encode(Fingerprint f, DurableOutput out) { - byte[] b = f.binary(); - encode(b, b.length, out); + encode(f.binary(), out); } public static Fingerprint decode(DurableInput in) { diff --git a/src/io/lacuna/bifurcan/durable/Rebases.java b/src/io/lacuna/bifurcan/durable/Rebases.java index 6950ec9..9f425b9 100644 --- a/src/io/lacuna/bifurcan/durable/Rebases.java +++ b/src/io/lacuna/bifurcan/durable/Rebases.java @@ -9,15 +9,16 @@ public class Rebases { public static void encode(IMap rebases, DurableOutput out) { out.writeUnsignedInt(rebases.size()); - rebases.entries().stream().sorted(Comparator.comparing(IEntry::key)).forEach( e -> { + rebases.entries().stream().sorted(Comparator.comparing(IEntry::key)).forEach(e -> { Fingerprints.encode(e.key(), out); Fingerprints.encode(e.value(), out); }); } public static IMap decode(DurableInput in) { - return Map.from( LongStream.range(0, in.readUnsignedInt()) - .mapToObj(n -> IEntry.of(Fingerprints.decode(in), Fingerprints.decode(in))) - .iterator()); + return Map.from( + LongStream.range(0, in.readUnsignedInt()) + .mapToObj(i -> IEntry.of(Fingerprints.decode(in), Fingerprints.decode(in))) + .iterator()); } } diff --git a/src/io/lacuna/bifurcan/durable/Roots.java b/src/io/lacuna/bifurcan/durable/Roots.java index 5d9034e..0a4bb79 100644 --- a/src/io/lacuna/bifurcan/durable/Roots.java +++ b/src/io/lacuna/bifurcan/durable/Roots.java @@ -64,6 +64,10 @@ public static DurableInput cachedInput(DurableInput in) { return new BufferInput((ByteBuffer) bytes.flip()); } + public static Root open(Path directory, Fingerprint fingerprint) { + return open(directory.resolve(fingerprint.toHexString() + ".bfn")); + } + public static Root open(Path path) { AtomicReference> fn = new AtomicReference<>(); fn.set(Functions.memoize(l -> open(path.getParent().resolve(l.fingerprint.toHexString() + ".bfn"), l.rebases, fn.get()))); @@ -86,8 +90,9 @@ private static Root open(Path path, IMap parentRebases // read in header Fingerprint fingerprint = Fingerprints.decode(file); - IMap rebases = parentRebases;// parentRebases.union(Rebases.decode(file)); - ISet dependencies = Dependencies.decode(file).stream().map(f -> rebases.get(f, f)).collect(Sets.collector()); + ISet rawDependencies = Dependencies.decode(file); + IMap rebases = parentRebases.union(Rebases.decode(file)); + ISet dependencies = rawDependencies.stream().map(f -> rebases.get(f, f)).collect(Sets.collector()); // mmap // final DurableInput.Pool contents = map(fc).slice(file.position(), size).pool(); @@ -160,6 +165,11 @@ public Root open(Fingerprint dependency) { public ISet dependencies() { return dependencies; } + + @Override + public String toString() { + return path.toString(); + } }; } catch (IOException e) { diff --git a/src/io/lacuna/bifurcan/durable/codecs/Util.java b/src/io/lacuna/bifurcan/durable/codecs/Core.java similarity index 53% rename from src/io/lacuna/bifurcan/durable/codecs/Util.java rename to src/io/lacuna/bifurcan/durable/codecs/Core.java index ece87b7..781ef6f 100644 --- a/src/io/lacuna/bifurcan/durable/codecs/Util.java +++ b/src/io/lacuna/bifurcan/durable/codecs/Core.java @@ -2,14 +2,42 @@ import io.lacuna.bifurcan.*; import io.lacuna.bifurcan.IDurableCollection.Fingerprint; +import io.lacuna.bifurcan.diffs.DiffMap; import io.lacuna.bifurcan.durable.BlockPrefix; import io.lacuna.bifurcan.durable.io.DurableBuffer; import io.lacuna.bifurcan.utils.Iterators; -public class Util { +import java.util.Comparator; +import java.util.Iterator; +import java.util.function.Function; +import java.util.function.Supplier; + +public class Core { private static ThreadLocal> COMPACT_SET = new ThreadLocal<>(); + public static T compacting(ISet compactSet, Supplier body) { + COMPACT_SET.set(compactSet); + try { + return body.get(); + } finally { + COMPACT_SET.set(null); + } + } + + /** + * Encodes a block to {@code out} + */ + public static void encodeBlock(IList os, IDurableEncoding encoding, DurableOutput out) { + if (os.size() == 1) { + encodeSingleton(os.first(), encoding, out); + } else if (encoding instanceof IDurableEncoding.Primitive) { + encodePrimitives(os, (IDurableEncoding.Primitive) encoding, out); + } else { + throw new IllegalArgumentException(String.format("cannot encode primitive with %s", encoding.description())); + } + } + /** * Encodes a block of one or more primitive objects */ @@ -21,25 +49,31 @@ public static void encodePrimitives(IList os, IDurableEncoding.Primitive * Encodes a singleton block, which may be a collection */ public static void encodeSingleton(Object o, IDurableEncoding encoding, DurableOutput out) { - if (COMPACT_SET.get() != null - && o instanceof IDurableCollection - && COMPACT_SET.get().contains(((IDurableCollection) o).root().fingerprint())) { - inlineSingleton(((IDurableCollection) o).bytes(), encoding, out); - - } else if (o instanceof IMap && encoding instanceof IDurableEncoding.Map) { - if (o instanceof IDiffMap && ((IDiffMap) o).underlying() instanceof IDurableCollection) { - IDiffMap m = (IDiffMap) o; - Diffs.encodeDiffHashMap(m, (IDurableCollection) m.underlying(), (IDurableEncoding.Map) encoding, out); + if (o instanceof IDurableCollection) { + ISet compactSet = COMPACT_SET.get(); + + // if it's within the scope of the compaction, inline the collection + if (compactSet != null && compactSet.contains(fingerprint(o))) { + inlineCollection((IDurableCollection) o, COMPACT_SET.get(), out); + + // otherwise just put in a reference } else { - HashMap.encodeSortedEntries(((IMap) o).hashSortedEntries(), (IDurableEncoding.Map) encoding, out); + Reference.from((IDurableCollection) o).encode(out); } + // some sort of map + } else if ((o instanceof IMap || o instanceof java.util.Map) && encoding instanceof IDurableEncoding.Map) { + encodeMap(o, (IDurableEncoding.Map) encoding, out); + + // some sort of set } else if (o instanceof ISet && encoding instanceof IDurableEncoding.Set) { throw new UnsupportedOperationException("sets are not yet supported"); + // some sort of list } else if (o instanceof IList && encoding instanceof IDurableEncoding.List) { io.lacuna.bifurcan.durable.codecs.List.encode(((IList) o).iterator(), (IDurableEncoding.List) encoding, out); + // a singleton primitive } else if (encoding instanceof IDurableEncoding.Primitive) { encodePrimitives(LinearList.of(o), (IDurableEncoding.Primitive) encoding, out); @@ -48,38 +82,52 @@ public static void encodeSingleton(Object o, IDurableEncoding encoding, DurableO } } - /** - * Encodes a block to {@code out} - */ - public static void encodeBlock(IList os, IDurableEncoding encoding, DurableOutput out) { - if (os.size() == 1) { - encodeSingleton(os.first(), encoding, out); - } else if (encoding instanceof IDurableEncoding.Primitive) { - encodePrimitives(os, (IDurableEncoding.Primitive) encoding, out); + public static void encodeMap(Object o, IDurableEncoding.Map encoding, DurableOutput out) { + // it's a diff over a durable collection + if (o instanceof IDiffMap && ((IDiffMap) o).underlying() instanceof IDurableCollection) { + IDiffMap m = (IDiffMap) o; + DiffHashMap.encodeDiffHashMap(m, (IDurableCollection) m.underlying(), encoding, out); + + // it's a Java map + } else if (o instanceof java.util.Map) { + Iterator> it = ((java.util.Map) o).entrySet().stream() + .map(e -> + IEntry.of( + encoding.keyEncoding().hashFn().applyAsLong(e.getKey()), + e.getKey(), + e.getValue())) + .sorted(Comparator.comparing(IEntry.WithHash::keyHash)) + .iterator(); + HashMap.encodeSortedEntries(it, encoding, out); + + // a normal map } else { - throw new IllegalArgumentException(String.format("cannot encode primitive with %s", encoding.description())); + HashMap.encodeSortedEntries(((IMap) o).hashSortedEntries(), encoding, out); } } - public static void inlineSingleton(DurableInput.Pool pool, IDurableEncoding encoding, DurableOutput out) { - DurableInput in = pool.instance(); - switch (in.readPrefix().type) { - case PRIMITIVE: - out.transferFrom(pool.instance()); - break; + public static void inlineCollection(IDurableCollection c, ISet compactSet, DurableOutput out) { + if (c instanceof IDiffMap) { + IList> stack = diffStack((IDiffMap) c, x -> + x.underlying() instanceof IDiffMap && compactSet.contains(fingerprint(x.underlying())) + ? (IDiffMap) x.underlying() + : null + ); - case REFERENCE: - - - case HASH_MAP: + // inline the entire map + if (compactSet.contains(fingerprint(stack.first().underlying()))) { + DiffHashMap.inline(stack, (IDurableEncoding.Map) c.encoding(), out); + // inline a set of diffs without inlining the underlying collection + } else { + DiffHashMap.inlineDiffs(stack, (IDurableEncoding.Map) c.encoding(), out); + } - case DIFF_HASH_MAP: - - case LIST: - - default: + } else if (c instanceof IMap.Durable) { + HashMap.inline(c.bytes(), (IDurableEncoding.Map) c.encoding(), c.root(), out); + } else { + throw new UnsupportedOperationException("don't know how to inline " + c.getClass().getSimpleName()); } } @@ -105,7 +153,7 @@ public static IDurableCollection decodeCollection(BlockPrefix prefix, IDurableEn } return prefix.type == BlockPrefix.BlockType.HASH_MAP ? HashMap.decode((IDurableEncoding.Map) encoding, root, pool) - : Diffs.decodeDiffHashMap((IDurableEncoding.Map) encoding, root, pool); + : DiffHashMap.decodeDiffHashMap((IDurableEncoding.Map) encoding, root, pool); case LIST: if (!(encoding instanceof IDurableEncoding.List)) { @@ -132,4 +180,27 @@ public static IDurableEncoding.SkippableIterator decodeBlock(DurableInput in, ID return Iterators.skippable(Iterators.singleton(decodeCollection(prefix, encoding, root, in.pool()))); } } + + + /// utility functions + + private static Fingerprint fingerprint(Object c) { + return ((IDurableCollection) c).root().fingerprint(); + } + + private static IList diffStack(T initial, Function inner) { + IList result = new LinearList<>(); + T curr = initial; + for (; ; ) { + result.addFirst(curr); + T next = inner.apply(curr); + if (next != null) { + curr = next; + } else { + break; + } + } + return result; + } + } diff --git a/src/io/lacuna/bifurcan/durable/codecs/Diffs.java b/src/io/lacuna/bifurcan/durable/codecs/DiffHashMap.java similarity index 60% rename from src/io/lacuna/bifurcan/durable/codecs/Diffs.java rename to src/io/lacuna/bifurcan/durable/codecs/DiffHashMap.java index 98835bc..5221f02 100644 --- a/src/io/lacuna/bifurcan/durable/codecs/Diffs.java +++ b/src/io/lacuna/bifurcan/durable/codecs/DiffHashMap.java @@ -4,11 +4,16 @@ import io.lacuna.bifurcan.durable.BlockPrefix; import io.lacuna.bifurcan.durable.BlockPrefix.BlockType; import io.lacuna.bifurcan.durable.io.DurableBuffer; +import io.lacuna.bifurcan.utils.Iterators; -import static io.lacuna.bifurcan.durable.codecs.Util.decodeCollection; +import java.util.Comparator; +import java.util.Iterator; +import static io.lacuna.bifurcan.diffs.Util.skipIndices; +import static io.lacuna.bifurcan.durable.codecs.Core.decodeCollection; -public class Diffs { + +public class DiffHashMap { public interface IDurableMap extends IDiffMap, io.lacuna.bifurcan.IMap.Durable { } @@ -23,13 +28,43 @@ public static void encodeDiffHashMap(IDiffMap m, IDurableCollection removed.flushTo(acc); // added entries - DurableMap.encode(m.added().entries().iterator(), encoding, (int) m.added().size(), acc); + HashMap.encodeSortedEntries(m.added().hashSortedEntries(), encoding, acc); // underlying Reference.encode(underlying, acc); }); } + public static void inlineDiffs(IList> diffStack, IDurableEncoding.Map encoding, DurableOutput out) { + DurableBuffer.flushTo(out, BlockType.DIFF_HASH_MAP, acc -> { + // removed indices + SkipTable.Writer removed = new SkipTable.Writer(); + IDiffMap.mergedRemovedIndices((IList) diffStack).forEachRemaining((long l) -> removed.append(l, 0)); + acc.writeUnsignedByte(removed.tiers()); + removed.flushTo(acc); + + // added entries + HashMap.encodeSortedEntries(IDiffMap.mergedAddedEntries(diffStack), encoding, acc); + + // underlying + Reference.encode((IDurableCollection) diffStack.first().underlying(), acc); + }); + } + + public static void inline(IList> diffStack, IDurableEncoding.Map encoding, DurableOutput out) { + Iterator> underlyingEntries = + skipIndices( + diffStack.first().underlying().hashSortedEntries(), + IDiffMap.mergedRemovedIndices((IList) diffStack)); + + Iterator> entries = + Iterators.mergeSort( + LinearList.of(underlyingEntries, IDiffMap.mergedAddedEntries(diffStack)), + Comparator.comparing(IEntry.WithHash::keyHash)); + + HashMap.encodeSortedEntries(entries, encoding, out); + } + public static IMap.Durable decodeDiffHashMap(IDurableEncoding.Map encoding, IDurableCollection.Root root, DurableInput.Pool bytes) { DurableInput in = bytes.instance(); @@ -59,7 +94,7 @@ public ISortedSet removedIndices() { } @Override - public IDurableEncoding encoding() { + public IDurableEncoding.Map encoding() { return encoding; } diff --git a/src/io/lacuna/bifurcan/durable/codecs/HashMap.java b/src/io/lacuna/bifurcan/durable/codecs/HashMap.java index d05c024..95595a8 100644 --- a/src/io/lacuna/bifurcan/durable/codecs/HashMap.java +++ b/src/io/lacuna/bifurcan/durable/codecs/HashMap.java @@ -208,6 +208,8 @@ public static DurableMap decode(IDurableEncoding.Map encoding, IDurableCollectio DurableInput.Pool entries = in.sliceBytes((pos + prefix.length) - in.position()).pool(); return new DurableMap(pool, root, size, hashTable, skipTable, entries, encoding); + + } /// inlining diff --git a/src/io/lacuna/bifurcan/durable/codecs/HashMapEntries.java b/src/io/lacuna/bifurcan/durable/codecs/HashMapEntries.java index 8c11832..634c71e 100644 --- a/src/io/lacuna/bifurcan/durable/codecs/HashMapEntries.java +++ b/src/io/lacuna/bifurcan/durable/codecs/HashMapEntries.java @@ -11,8 +11,8 @@ import java.util.PrimitiveIterator; import java.util.function.BiPredicate; -import static io.lacuna.bifurcan.durable.codecs.Util.decodeBlock; -import static io.lacuna.bifurcan.durable.codecs.Util.encodeBlock; +import static io.lacuna.bifurcan.durable.codecs.Core.decodeBlock; +import static io.lacuna.bifurcan.durable.codecs.Core.encodeBlock; /** * A block that represents zero or more key/value pairs in a HashMap. diff --git a/src/io/lacuna/bifurcan/durable/codecs/List.java b/src/io/lacuna/bifurcan/durable/codecs/List.java index ef4089f..02a2c21 100644 --- a/src/io/lacuna/bifurcan/durable/codecs/List.java +++ b/src/io/lacuna/bifurcan/durable/codecs/List.java @@ -8,7 +8,7 @@ import java.util.Iterator; -import static io.lacuna.bifurcan.durable.codecs.Util.encodeBlock; +import static io.lacuna.bifurcan.durable.codecs.Core.encodeBlock; /** * An indexed list, encoded as: diff --git a/src/io/lacuna/bifurcan/durable/codecs/Reference.java b/src/io/lacuna/bifurcan/durable/codecs/Reference.java index 6c53e51..2747031 100644 --- a/src/io/lacuna/bifurcan/durable/codecs/Reference.java +++ b/src/io/lacuna/bifurcan/durable/codecs/Reference.java @@ -20,8 +20,10 @@ private Reference(Fingerprint fingerprint, long position) { this.position = position; } - public static Reference from(IDurableCollection collection) { - return new Reference(collection.root().fingerprint(), collection.bytes().instance().position()); + public static Reference from(IDurableCollection c) { + // get the offset from the start of `c` to the start of the root collection + long pos = c.bytes().instance().bounds().absolute().start - c.root().bytes().instance().bounds().absolute().start; + return new Reference(c.root().fingerprint(), pos); } public void encode(DurableOutput out) { @@ -32,10 +34,12 @@ public void encode(DurableOutput out) { }); } + public DurableInput.Pool underlyingBytes(IDurableCollection.Root root) { + return root.open(fingerprint).bytes().instance().seek(position).slicePrefixedBlock().pool(); + } + public IDurableCollection decodeCollection(IDurableEncoding encoding, IDurableCollection.Root root) { - IDurableCollection.Root underlyingRoot = root.open(fingerprint); - DurableInput.Pool underlyingBytes = underlyingRoot.bytes().instance().seek(position).slicePrefixedBlock().pool(); - return Util.decodeCollection(encoding, underlyingRoot, underlyingBytes); + return Core.decodeCollection(encoding, root.open(fingerprint), underlyingBytes(root)); } public static void encode(IDurableCollection collection, DurableOutput out) { diff --git a/src/io/lacuna/bifurcan/durable/codecs/SkipTable.java b/src/io/lacuna/bifurcan/durable/codecs/SkipTable.java index d354922..15c4015 100644 --- a/src/io/lacuna/bifurcan/durable/codecs/SkipTable.java +++ b/src/io/lacuna/bifurcan/durable/codecs/SkipTable.java @@ -347,6 +347,12 @@ public long flushTo(DurableOutput out) { }); return out.written() - offset; } + + public ISortedMap toOffHeapMap() { + DurableBuffer tmp = new DurableBuffer(); + flushTo(tmp); + return SkipTable.decode(tmp.toOffHeapInput().sliceBlock(BlockType.TABLE).pool(), tiers()); + } } } diff --git a/src/io/lacuna/bifurcan/durable/codecs/TempStream.java b/src/io/lacuna/bifurcan/durable/codecs/TempStream.java index 8f08c0e..ab370b7 100644 --- a/src/io/lacuna/bifurcan/durable/codecs/TempStream.java +++ b/src/io/lacuna/bifurcan/durable/codecs/TempStream.java @@ -8,8 +8,8 @@ import java.util.Iterator; -import static io.lacuna.bifurcan.durable.codecs.Util.decodeBlock; -import static io.lacuna.bifurcan.durable.codecs.Util.encodeBlock; +import static io.lacuna.bifurcan.durable.codecs.Core.decodeBlock; +import static io.lacuna.bifurcan.durable.codecs.Core.encodeBlock; /** diff --git a/src/io/lacuna/bifurcan/durable/io/DurableBuffer.java b/src/io/lacuna/bifurcan/durable/io/DurableBuffer.java index d86703e..ee6365a 100644 --- a/src/io/lacuna/bifurcan/durable/io/DurableBuffer.java +++ b/src/io/lacuna/bifurcan/durable/io/DurableBuffer.java @@ -53,6 +53,15 @@ public DurableInput toInput() { return DurableInput.from(flushed.stream().map(IBuffer::toInput).collect(Lists.linearCollector())); } + public DurableInput toOffHeapInput() { + assert (written() <= Integer.MAX_VALUE); + DurableInput tmp = toInput(); + ByteBuffer b = Bytes.allocate((int) tmp.remaining()); + tmp.read(b); + tmp.close(); + return new BufferInput((ByteBuffer) b.flip()); + } + public IList toBuffers() { close(); return flushed; diff --git a/src/io/lacuna/bifurcan/durable/io/FileOutput.java b/src/io/lacuna/bifurcan/durable/io/FileOutput.java index 1a364b0..6d5dc73 100644 --- a/src/io/lacuna/bifurcan/durable/io/FileOutput.java +++ b/src/io/lacuna/bifurcan/durable/io/FileOutput.java @@ -1,12 +1,8 @@ package io.lacuna.bifurcan.durable.io; +import io.lacuna.bifurcan.*; import io.lacuna.bifurcan.IDurableCollection.Fingerprint; -import io.lacuna.bifurcan.IMap; -import io.lacuna.bifurcan.ISet; -import io.lacuna.bifurcan.durable.Bytes; -import io.lacuna.bifurcan.durable.Dependencies; -import io.lacuna.bifurcan.durable.Fingerprints; -import io.lacuna.bifurcan.durable.Rebases; +import io.lacuna.bifurcan.durable.*; import io.lacuna.bifurcan.durable.allocator.GenerationalAllocator; import io.lacuna.bifurcan.durable.allocator.IBuffer; @@ -19,6 +15,7 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.security.MessageDigest; +import java.util.function.Consumer; /** * A special {@link WritableByteChannel} for a durable collection file, which will construct a hash as the file is @@ -27,8 +24,6 @@ public class FileOutput implements WritableByteChannel { // `version` encompasses the algorithm, but not necessarily the number of bytes we use - private static final String ALGORITHM = "SHA-512"; - private static final int HASH_BYTES = 32; // these will be the first four bytes of any valid collection file public static final ByteBuffer MAGIC_BYTES = (ByteBuffer) @@ -39,19 +34,19 @@ public class FileOutput implements WritableByteChannel { .put((byte) 0x01) // version .flip(); - private static final int PREFIX_LENGTH = MAGIC_BYTES.remaining() + 1 + HASH_BYTES; + private static final int PREFIX_LENGTH = MAGIC_BYTES.remaining() + 1 + Fingerprint.HASH_BYTES; private final Path path; private final MessageDigest digest; private final FileChannel file; private final IBuffer buffer; - private byte[] hash; + private Fingerprint fingerprint; - public FileOutput(ISet dependencies, IMap rebases) { + private FileOutput(ISet dependencies, IMap rebases) { try { this.path = Files.createTempFile("bifurcan-", ".draft"); - this.digest = MessageDigest.getInstance(ALGORITHM); + this.digest = MessageDigest.getInstance(Fingerprint.ALGORITHM); this.buffer = GenerationalAllocator.allocate(16 << 20); this.file = FileChannel.open(path, StandardOpenOption.WRITE); @@ -61,7 +56,7 @@ public FileOutput(ISet dependencies, IMap file.position(PREFIX_LENGTH); ByteChannelOutput.wrap(this, out -> { Dependencies.encode(dependencies, out); -// Rebases.encode(rebases, out); + Rebases.encode(rebases, out); }); } catch (Exception e) { @@ -69,6 +64,21 @@ public FileOutput(ISet dependencies, IMap } } + public static Fingerprint write(Path directory, IMap rebases, Consumer body) { + Dependencies.enter(); + DurableBuffer acc = new DurableBuffer(); + + body.accept(acc); + + FileOutput file = new FileOutput(Dependencies.exit(), rebases); + DurableOutput out = DurableOutput.from(file); + acc.flushTo(out); + out.close(); + + file.moveTo(directory); + return file.fingerprint; + } + /** * Finalizes the collection after {@link #close()} is called, moving the file to {@code directory}, with the hex-encoded * hash for a name. @@ -77,9 +87,9 @@ public Path moveTo(Path directory) { assert !isOpen(); try { - String hexHash = Bytes.toHexString(ByteBuffer.wrap(hash, 0, HASH_BYTES)); - Path newPath = directory.resolve(hexHash + ".bfn"); + Path newPath = directory.resolve(fingerprint.toHexString() + ".bfn"); directory.toFile().mkdirs(); + // TODO: should we actually replace a file with the same hash? Files.move(path, newPath, StandardCopyOption.REPLACE_EXISTING); return newPath; } catch (IOException e) { @@ -122,7 +132,7 @@ public boolean isOpen() { @Override public void close() { - this.hash = digest.digest(); + this.fingerprint = Fingerprints.from(Fingerprints.trim(digest.digest(), Fingerprint.HASH_BYTES)); this.buffer.free(); try { @@ -132,7 +142,7 @@ public void close() { ByteChannelOutput.wrap(this, out -> DurableBuffer.flushTo(out, acc -> { acc.write(MAGIC_BYTES.duplicate()); - Fingerprints.encode(hash, HASH_BYTES, acc); + Fingerprints.encode(fingerprint, acc); })); file.close(); diff --git a/test/bifurcan/durable_test.clj b/test/bifurcan/durable_test.clj index f1430dc..05e04ed 100644 --- a/test/bifurcan/durable_test.clj +++ b/test/bifurcan/durable_test.clj @@ -4,6 +4,7 @@ [byte-streams :as bs] [clojure.edn :as edn] [clojure.java.io :as io] + [clojure.java.shell :as sh] [clojure.test :refer :all] [clojure.test.check.generators :as gen] [clojure.test.check.properties :as prop] @@ -16,8 +17,11 @@ [io.lacuna.bifurcan ICollection IDurableCollection + IDurableCollection$Root + IDurableCollection$Rebase IMap Map + Set List Maps IEntry @@ -51,13 +55,16 @@ (set! *warn-on-reflection* true) -(def ^java.nio.file.Path test-dir (.toPath (io/file "/tmp/bifurcan-tests"))) +(def test-dirs + (->> [:repl :maps :sets :lists] + (map (fn [t] + (let [f (io/file (str "/tmp/bifurcan-tests/" (name t)))] + (.mkdirs f) + [t (.toPath f)]))) + (into {}))) -(-> test-dir .toFile .mkdirs) - -(defn clear-test-dir [] - (doseq [^java.io.File f (->> test-dir .toFile .listFiles)] - (.delete f))) +(defn clear-test-dir [t] + (sh/sh "rm" (str (test-dirs t) "/*"))) (def gen-pos-int (gen/such-that @@ -159,15 +166,8 @@ (defn create-skip-table [entries] (let [writer (SkipTable$Writer.) _ (doseq [[index offset] entries] - (.append writer index offset)) - out (DurableBuffer.) - _ (.flushTo writer out) - in (.toInput out) - contents (.sliceBlock in BlockPrefix$BlockType/TABLE) - buf (Bytes/allocate (.size contents)) - _ (.read contents buf) - _ (free! in)] - (SkipTable/decode (.pool (BufferInput. (.flip buf))) (.tiers writer)))) + (.append writer index offset))] + (.toOffHeapMap writer))) (defn print-skip-table [^DurableInput in] (->> (repeatedly #(when (pos? (.remaining in)) (.readUVLQ in))) @@ -200,10 +200,11 @@ ;;; DurableMap -(defn save [^ICollection c] - (let [^IDurableCollection c' (.save c edn-encoding test-dir)] - (-> c' .root .path .toFile .deleteOnExit) - c')) +(defn save + ([c] + (save (:repl test-dirs) c)) + ([dir ^ICollection c] + (.save c edn-encoding dir))) (def durable-map-actions (-> coll/map-actions @@ -212,16 +213,41 @@ (def bifurcan-durable-map (assoc coll/bifurcan-map - :save save)) + :save #(save (test-dirs :maps) %))) -(u/def-collection-check test-durable-map 1e6 durable-map-actions +(u/def-collection-check test-durable-map iterations durable-map-actions [] [m (Map.) bifurcan-durable-map m' {} coll/clj-map] (try (coll/map= m' m) (finally - (clear-test-dir)))) + (clear-test-dir :maps)))) + +(defn dep-chain [^IDurableCollection$Root r] + (cons (.fingerprint r) + (let [deps (->> r .dependencies .iterator iterator-seq (map #(.open r %)))] + (when-not (empty? deps) + (dep-chain (first deps)))))) + +(defn compact [^IDurableCollection c deps] + (if (< (count deps) 2) + c + (let [c' (-> c .root (.open (first deps)) (.decode edn-encoding)) + r (.compact c' (-> ^Iterable deps .iterator Set/from))] + (.apply r c)))) + +(u/def-collection-check test-durable-map-compact iterations durable-map-actions + [n-drop gen-small-pos-int + n-take gen-small-pos-int] + [m (Map.) bifurcan-durable-map] + (let [m (save (:maps test-dirs) m) + deps (->> ^IDurableCollection m .root dep-chain (drop n-drop) (take n-take))] + (if (= m (compact m deps)) + true + (do + (prn m (compact m deps)) + false)))) ;;; DurableList @@ -232,7 +258,7 @@ (def bifurcan-durable-list (assoc coll/bifurcan-list - :save save)) + :save #(save (test-dirs :lists) %))) (u/def-collection-check test-durable-list iterations durable-list-actions [] @@ -241,24 +267,4 @@ (try (coll/list= m' m) (finally - (clear-test-dir)))) - - #_(defspec test-durable-list iterations - (prop/for-all [l (coll/list-gen #(List.))] - (let [out (DurableBuffer.) - _ (DurableList/encode (.iterator ^Iterable l) edn-encoding out) - in (.toInput out) - l' (DurableList/decode edn-encoding nil (.pool in))] - (and - (= l l') - (free! in))))) - - -(comment - (def p (.toPath (io/file "/tmp"))) - (-> (Map.) - (.put 1 1) - (.save edn-encoding p) - (.put 2 2) - (.put 1 42) - (.save edn-encoding p))) + (clear-test-dir :lists))))