Merge 0.7.0 into master

.github/workflows/build.yml

@@ -3,15 +3,15 @@ name: HNSW CI
 on: [push, pull_request]
 
 jobs:
-  test:
+  test_python:
     runs-on: ${{matrix.os}}
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: ['3.6', '3.7', '3.8', '3.9']
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -20,3 +20,45 @@ jobs:
 
       - name: Test
         run: python -m unittest discover --start-directory python_bindings/tests --pattern "*_test*.py"
+
+  test_cpp:
+    runs-on: ${{matrix.os}}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Build
+        run: |
+          mkdir build
+          cd build
+          cmake ..
+          if [ "$RUNNER_OS" == "Linux" ]; then
+            make
+          elif [ "$RUNNER_OS" == "Windows" ]; then
+            cmake --build ./ --config Release
+          fi
+        shell: bash
+
+      - name: Prepare test data
+        run: |
+          pip install numpy
+          cd examples
+          python update_gen_data.py
+        shell: bash
+
+      - name: Test
+        run: |
+          cd build
+          if [ "$RUNNER_OS" == "Windows" ]; then
+            cp ./Release/* ./
+          fi
+          ./searchKnnCloserFirst_test
+          ./searchKnnWithFilter_test
+          ./test_updates
+          ./test_updates update
+        shell: bash

CMakeLists.txt

@@ -22,6 +22,9 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
     add_executable(searchKnnCloserFirst_test examples/searchKnnCloserFirst_test.cpp)
     target_link_libraries(searchKnnCloserFirst_test hnswlib)
 
+    add_executable(searchKnnWithFilter_test examples/searchKnnWithFilter_test.cpp)
+    target_link_libraries(searchKnnWithFilter_test hnswlib)
+
     add_executable(main main.cpp sift_1b.cpp)
     target_link_libraries(main hnswlib)
 endif()

README.md

@@ -247,7 +247,7 @@ Please make pull requests against the `develop` branch.
 
 When making changes please run tests (and please add a test to `python_bindings/tests` in case there is new functionality):
 ```bash
-python -m unittest discover --start-directory python_bindings/tests --pattern "*_test*.py
+python -m unittest discover --start-directory python_bindings/tests --pattern "*_test*.py"
 ```
 
 
@@ -267,7 +267,7 @@ https://github.com/dbaranchuk/ivf-hnsw
 * Java bindings using Java Native Access: https://github.com/stepstone-tech/hnswlib-jna
 * .Net implementation: https://github.com/microsoft/HNSW.Net
 * CUDA implementation: https://github.com/js1010/cuhnsw
-
+* Rust implementation for memory and thread safety purposes and There is  A Trait to enable the user to implement its own distances. It takes as data slices of types T satisfying T:Serialize+Clone+Send+Sync.: https://github.com/jean-pierreBoth/hnswlib-rs
 
 ### 200M SIFT test reproduction 
 To download and extract the bigann dataset (from root directory):

TESTING_RECALL.md

@@ -27,7 +27,7 @@ max_elements defines the maximum number of elements that can be stored in the st
 
 ### measuring recall example
 
-```
+```python
 import hnswlib
 import numpy as np
 

examples/git_tester.py

@@ -1,34 +1,40 @@
+import os
+import shutil
+
+from sys import platform
 from pydriller import Repository
-import os 
-import datetime
-os.system("cp examples/speedtest.py examples/speedtest2.py") # the file has to be outside of git
-for idx, commit in enumerate(Repository('.', from_tag="v0.6.0").traverse_commits()):    
-    name=commit.msg.replace('\n', ' ').replace('\r', ' ')
-    print(idx, commit.hash, name)
 
 
+speedtest_src_path = os.path.join("examples", "speedtest.py")
+speedtest_copy_path = os.path.join("examples", "speedtest2.py")
+shutil.copyfile(speedtest_src_path, speedtest_copy_path) # the file has to be outside of git
+
+commits = list(Repository('.', from_tag="v0.6.0").traverse_commits())
+print("Found commits:")
+for idx, commit in enumerate(commits):
+    name = commit.msg.replace('\n', ' ').replace('\r', ' ')
+    print(idx, commit.hash, name)
 
-for commit in Repository('.', from_tag="v0.6.0").traverse_commits():
-
-    name=commit.msg.replace('\n', ' ').replace('\r', ' ')
-    print(commit.hash, name)
-
-    os.system(f"git checkout {commit.hash}; rm -rf build; ")
+for commit in commits:
+    name = commit.msg.replace('\n', ' ').replace('\r', ' ')
+    print("\nProcessing", commit.hash, name)
+
+    if os.path.exists("build"):
+        shutil.rmtree("build")
+    os.system(f"git checkout {commit.hash}")
     print("\n\n--------------------\n\n")
-    ret=os.system("python -m pip install .")
-    print(ret)
-
-    if ret != 0:
-        print ("build failed!!!!")
-        print ("build failed!!!!")
-        print ("build failed!!!!")
-        print ("build failed!!!!")
-        continue    
-
-    os.system(f'python examples/speedtest2.py -n "{name}" -d 4 -t 1')
-    os.system(f'python examples/speedtest2.py -n "{name}" -d 64 -t 1')
-    os.system(f'python examples/speedtest2.py -n "{name}" -d 128 -t 1')
-    os.system(f'python examples/speedtest2.py -n "{name}" -d 4 -t 24')
-    os.system(f'python examples/speedtest2.py -n "{name}" -d 128 -t 24')
+    ret = os.system("python -m pip install .")
+    print("Install result:", ret)
 
+    if ret != 0:
+        print("build failed!!!!")
+        print("build failed!!!!")
+        print("build failed!!!!")
+        print("build failed!!!!")
+        continue
 
+    os.system(f'python {speedtest_copy_path} -n "{name}" -d 4 -t 1')
+    os.system(f'python {speedtest_copy_path} -n "{name}" -d 64 -t 1')
+    os.system(f'python {speedtest_copy_path} -n "{name}" -d 128 -t 1')
+    os.system(f'python {speedtest_copy_path} -n "{name}" -d 4 -t 24')
+    os.system(f'python {speedtest_copy_path} -n "{name}" -d 128 -t 24')

examples/searchKnnWithFilter_test.cpp

@@ -0,0 +1,173 @@
+// This is a test file for testing the filtering feature
+
+#include "../hnswlib/hnswlib.h"
+
+#include <assert.h>
+
+#include <vector>
+#include <iostream>
+
+namespace
+{
+
+using idx_t = hnswlib::labeltype;
+
+bool pickIdsDivisibleByThree(unsigned int label_id) {
+    return label_id % 3 == 0;
+}
+
+bool pickIdsDivisibleBySeven(unsigned int label_id) {
+    return label_id % 7 == 0;
+}
+
+bool pickNothing(unsigned int label_id) {
+    return false;
+}
+
+template<typename filter_func_t>
+void test_some_filtering(filter_func_t& filter_func, size_t div_num, size_t label_id_start) {
+    int d = 4;
+    idx_t n = 100;
+    idx_t nq = 10;
+    size_t k = 10;
+
+    std::vector<float> data(n * d);
+    std::vector<float> query(nq * d);
+
+    std::mt19937 rng;
+    rng.seed(47);
+    std::uniform_real_distribution<> distrib;
+
+    for (idx_t i = 0; i < n * d; ++i) {
+        data[i] = distrib(rng);
+    }
+    for (idx_t i = 0; i < nq * d; ++i) {
+        query[i] = distrib(rng);
+    }
+
+    hnswlib::L2Space space(d);
+    hnswlib::AlgorithmInterface<float,filter_func_t>* alg_brute  = new hnswlib::BruteforceSearch<float,filter_func_t>(&space, 2 * n);
+    hnswlib::AlgorithmInterface<float,filter_func_t>* alg_hnsw = new hnswlib::HierarchicalNSW<float,filter_func_t>(&space, 2 * n);
+
+    for (size_t i = 0; i < n; ++i) {
+        // `label_id_start` is used to ensure that the returned IDs are labels and not internal IDs
+        alg_brute->addPoint(data.data() + d * i, label_id_start + i);
+        alg_hnsw->addPoint(data.data() + d * i, label_id_start + i);
+    }
+
+    // test searchKnnCloserFirst of BruteforceSearch with filtering
+    for (size_t j = 0; j < nq; ++j) {
+        const void* p = query.data() + j * d;
+        auto gd = alg_brute->searchKnn(p, k, filter_func);
+        auto res = alg_brute->searchKnnCloserFirst(p, k, filter_func);
+        assert(gd.size() == res.size());
+        size_t t = gd.size();
+        while (!gd.empty()) {
+            assert(gd.top() == res[--t]);
+            assert((gd.top().second % div_num) == 0);
+            gd.pop();
+        }
+    }
+
+    // test searchKnnCloserFirst of hnsw with filtering
+    for (size_t j = 0; j < nq; ++j) {
+        const void* p = query.data() + j * d;
+        auto gd = alg_hnsw->searchKnn(p, k, filter_func);
+        auto res = alg_hnsw->searchKnnCloserFirst(p, k, filter_func);
+        assert(gd.size() == res.size());
+        size_t t = gd.size();
+        while (!gd.empty()) {
+            assert(gd.top() == res[--t]);
+            assert((gd.top().second % div_num) == 0);
+            gd.pop();
+        }
+    }
+
+    delete alg_brute;
+    delete alg_hnsw;
+}
+
+template<typename filter_func_t>
+void test_none_filtering(filter_func_t& filter_func, size_t label_id_start) {
+    int d = 4;
+    idx_t n = 100;
+    idx_t nq = 10;
+    size_t k = 10;
+
+    std::vector<float> data(n * d);
+    std::vector<float> query(nq * d);
+
+    std::mt19937 rng;
+    rng.seed(47);
+    std::uniform_real_distribution<> distrib;
+
+    for (idx_t i = 0; i < n * d; ++i) {
+        data[i] = distrib(rng);
+    }
+    for (idx_t i = 0; i < nq * d; ++i) {
+        query[i] = distrib(rng);
+    }
+
+    hnswlib::L2Space space(d);
+    hnswlib::AlgorithmInterface<float,filter_func_t>* alg_brute  = new hnswlib::BruteforceSearch<float,filter_func_t>(&space, 2 * n);
+    hnswlib::AlgorithmInterface<float,filter_func_t>* alg_hnsw = new hnswlib::HierarchicalNSW<float,filter_func_t>(&space, 2 * n);
+
+    for (size_t i = 0; i < n; ++i) {
+        // `label_id_start` is used to ensure that the returned IDs are labels and not internal IDs
+        alg_brute->addPoint(data.data() + d * i, label_id_start + i);
+        alg_hnsw->addPoint(data.data() + d * i, label_id_start + i);
+    }
+
+    // test searchKnnCloserFirst of BruteforceSearch with filtering
+    for (size_t j = 0; j < nq; ++j) {
+        const void* p = query.data() + j * d;
+        auto gd = alg_brute->searchKnn(p, k, filter_func);
+        auto res = alg_brute->searchKnnCloserFirst(p, k, filter_func);
+        assert(gd.size() == res.size());
+        assert(0 == gd.size());
+    }
+
+    // test searchKnnCloserFirst of hnsw with filtering
+    for (size_t j = 0; j < nq; ++j) {
+        const void* p = query.data() + j * d;
+        auto gd = alg_hnsw->searchKnn(p, k, filter_func);
+        auto res = alg_hnsw->searchKnnCloserFirst(p, k, filter_func);
+        assert(gd.size() == res.size());
+        assert(0 == gd.size());
+    }
+
+    delete alg_brute;
+    delete alg_hnsw;
+}
+
+} // namespace
+
+class CustomFilterFunctor: public hnswlib::FilterFunctor {
+    std::unordered_set<unsigned int> allowed_values;
+
+public:
+    explicit CustomFilterFunctor(const std::unordered_set<unsigned int>& values) : allowed_values(values) {}
+
+    bool operator()(unsigned int id) {
+        return allowed_values.count(id) != 0;
+    }
+};
+
+int main() {
+    std::cout << "Testing ..." << std::endl;
+
+    // some of the elements are filtered
+    test_some_filtering(pickIdsDivisibleByThree, 3, 17);
+    test_some_filtering(pickIdsDivisibleBySeven, 7, 17);
+
+    // all of the elements are filtered
+    test_none_filtering(pickNothing, 17);
+
+    // functor style which can capture context
+    CustomFilterFunctor pickIdsDivisibleByThirteen({26, 39, 52, 65});
+    test_some_filtering(pickIdsDivisibleByThirteen, 13, 21);
+
+    std::cout << "Test ok" << std::endl;
+
+    return 0;
+}