add symbolic link support for snapshot

pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>com.github.wenweihu86.raft</groupId>
     <artifactId>raft-java-all</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0</version>
     <packaging>pom</packaging>
 
     <name>raft-java-all</name>

raft-java-admin/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.github.wenweihu86.raft</groupId>
     <artifactId>raft-java-admin</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0</version>
     <packaging>jar</packaging>
 
     <name>raft-java-admin</name>

raft-java-admin/start_admin.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
-java -cp dependency/*:raft-java-admin-1.3.0.jar com.github.wenweihu86.raft.admin.AdminMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" conf get
+java -cp dependency/*:raft-java-admin-1.4.0.jar com.github.wenweihu86.raft.admin.AdminMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" conf get
 
-java -cp dependency/*:raft-java-admin-1.3.0.jar com.github.wenweihu86.raft.admin.AdminMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" conf add "127.0.0.1:8054:4,127.0.0.1:8055:5"
+java -cp dependency/*:raft-java-admin-1.4.0.jar com.github.wenweihu86.raft.admin.AdminMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" conf add "127.0.0.1:8054:4,127.0.0.1:8055:5"
 
-java -cp dependency/*:raft-java-admin-1.3.0.jar com.github.wenweihu86.raft.admin.AdminMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" conf del "127.0.0.1:8054:4,127.0.0.1:8055:5"
+java -cp dependency/*:raft-java-admin-1.4.0.jar com.github.wenweihu86.raft.admin.AdminMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" conf del "127.0.0.1:8054:4,127.0.0.1:8055:5"

raft-java-core/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>com.github.wenweihu86.raft</groupId>
     <artifactId>raft-java-core</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0</version>
     <packaging>jar</packaging>
 
     <name>raft-java-core</name>

raft-java-core/src/main/java/com/github/wenweihu86/raft/RaftNode.java

@@ -100,7 +100,9 @@ public RaftNode(List<RaftMessage.Server> servers, RaftMessage.Server localServer
             }
         }
         lastAppliedIndex = commitIndex;
+    }
 
+    public void init() {
         for (RaftMessage.Server server : configuration.getServersList()) {
             if (!peerMap.containsKey(server.getServerId())
                     && server.getServerId() != localServer.getServerId()) {
@@ -124,9 +126,6 @@ public void run() {
                 takeSnapshot();
             }
         }, RaftOptions.snapshotPeriodSeconds, RaftOptions.snapshotPeriodSeconds, TimeUnit.SECONDS);
-    }
-
-    public void init() {
         // start election
         resetElectionTimer();
     }
@@ -662,7 +661,7 @@ public void stepDown(long newTerm) {
         resetElectionTimer();
     }
 
-    private void takeSnapshot() {
+    public void takeSnapshot() {
         if (!snapshot.getIsInSnapshot().compareAndSet(false, true)) {
             LOG.info("already in snapshot, ignore takeSnapshot");
             return;

raft-java-core/src/main/java/com/github/wenweihu86/raft/storage/SegmentedLog.java

@@ -170,6 +170,7 @@ public void truncatePrefix(long newFirstIndex) {
                 try {
                     RaftFileUtils.closeFile(segment.getRandomAccessFile());
                     FileUtils.forceDelete(oldFile);
+                    totalSize -= segment.getFileSize();
                     startLogIndexSegmentMap.remove(segment.getStartIndex());
                 } catch (Exception ex2) {
                     LOG.warn("delete file exception:", ex2);

raft-java-core/src/main/java/com/github/wenweihu86/raft/storage/Snapshot.java

@@ -11,6 +11,8 @@
 import java.io.IOException;
 import java.io.RandomAccessFile;
 
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -60,9 +62,18 @@ public void reload() {
         }
     }
 
+    // 如果是软链接，需要打开实际文件句柄
     public TreeMap<String, SnapshotDataFile> readSnapshotDataFiles() {
         TreeMap<String, SnapshotDataFile> snapshotDataFileMap = new TreeMap<>();
         String snapshotDataDir = snapshotDir + File.separator + "data";
+        try {
+            Path snapshotDataPath = FileSystems.getDefault().getPath(snapshotDataDir);
+            snapshotDataPath = snapshotDataPath.toRealPath();
+            snapshotDataDir = snapshotDataPath.toString();
+        } catch (IOException ex) {
+            LOG.warn("readSnapshotDataFiles exception:", ex);
+            throw new RuntimeException(ex);
+        }
         List<String> fileNames = RaftFileUtils.getSortedFilesInDirectory(snapshotDataDir);
         for (String fileName : fileNames) {
             RandomAccessFile randomAccessFile = RaftFileUtils.openFile(snapshotDataDir, fileName, "r");

raft-java-core/src/test/java/com/github/wenweihu86/raft/storage/SnapshotTest.java

@@ -0,0 +1,41 @@
+package com.github.wenweihu86.raft.storage;
+
+import com.github.wenweihu86.raft.RaftOptions;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.TreeMap;
+
+/**
+ * Created by wenweihu86 on 2017/7/2.
+ */
+public class SnapshotTest {
+
+    @Test
+    public void testReadSnapshotDataFiles() throws IOException {
+        RaftOptions.dataDir = "./data";
+        File file = new File("./data/message");
+        file.mkdirs();
+        File file1 = new File("./data/message/queue1.txt");
+        file1.createNewFile();
+        File file2 = new File("./data/message/queue2.txt");
+        file2.createNewFile();
+
+        File snapshotFile = new File("./data/snapshot");
+        snapshotFile.mkdirs();
+        Path link = FileSystems.getDefault().getPath("./data/snapshot/data");
+        Path target = FileSystems.getDefault().getPath("./data/message").toRealPath();
+        Files.createSymbolicLink(link, target);
+
+        Snapshot snapshot = new Snapshot();
+        TreeMap<String, Snapshot.SnapshotDataFile> snapshotFileMap = snapshot.readSnapshotDataFiles();
+        System.out.println(snapshotFileMap.keySet());
+        Assert.assertTrue(snapshotFileMap.size() == 2);
+        Assert.assertTrue(snapshotFileMap.firstKey().equals("queue1.txt"));
+    }
+}

raft-java-example/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>com.github.wenweihu86.raft</groupId>
     <artifactId>raft-java-example</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0</version>
     <packaging>jar</packaging>
 
     <name>raft-java-example</name>

raft-java-example/start_client.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 
-java -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.client.ClientMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" hello raft
+java -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.client.ClientMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053" hello raft
 
-java -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.client.ConcurrentClientMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053"
+java -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.client.ConcurrentClientMain "127.0.0.1:8051,127.0.0.1:8052,127.0.0.1:8053"

raft-java-example/start_server.sh

@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
 
-java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example1/data -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8051:1"
+java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example1/data -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8051:1"
 
-java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example2/data -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8052:2"
+java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example2/data -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8052:2"
 
-java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example3/data -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8053:3"
+java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example3/data -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8053:3"
 
-java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example4/data -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8054:4"
+java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example4/data -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8054:4"
 
-java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example5/data -cp dependency/*:raft-java-example-1.3.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8055:5"
+java -Dcom.github.wenweihu86.raft.data.dir=/Users/baidu/local/raft-java-example5/data -cp dependency/*:raft-java-example-1.4.0.jar com.github.wenweihu86.raft.example.server.ServerMain "127.0.0.1:8051:1,127.0.0.1:8052:2,127.0.0.1:8053:3" "127.0.0.1:8055:5"