Implement ClickHouse Support (langchain-ai#3342)

* Implement ClickHouse Support Co-Authored-By: CalebZhang <[email protected]> Co-Authored-By: Divyansh Kachchhava <[email protected]> Co-Authored-By: Alfred Tze-Hong Ha <[email protected]> * Update ClickHouse client dependency Co-Authored-By: Divyansh Kachchhava <[email protected]> Co-Authored-By: CalebZhang <[email protected]> Co-Authored-By: Alfred Tze-Hong Ha <[email protected]> * Update int test to use test.skip() * Fix SQL injection risk * Add peer deps docs * fix yarn lint issue --------- Co-authored-by: CalebZhang <[email protected]> Co-authored-by: Divyansh Kachchhava <[email protected]> Co-authored-by: Alfred Tze-Hong Ha <[email protected]> Co-authored-by: jacoblee93 <[email protected]> Co-authored-by: CalebZhang <[email protected]>
shareefmorayur · Nov 28, 2023 · 85470e4 · 85470e4
1 parent 8684351
commit 85470e4
Show file tree

Hide file tree

Showing 16 changed files with 583 additions and 19 deletions.
diff --git a/docs/api_refs/typedoc.json b/docs/api_refs/typedoc.json
@@ -91,6 +91,7 @@
     "./langchain/src/llms/fake.ts",
     "./langchain/src/prompts/index.ts",
     "./langchain/src/prompts/load.ts",
+    "./langchain/src/vectorstores/clickhouse.ts",
     "./langchain/src/vectorstores/analyticdb.ts",
     "./langchain/src/vectorstores/base.ts",
     "./langchain/src/vectorstores/cassandra.ts",

diff --git a/docs/core_docs/docs/integrations/vectorstores/clickhouse.mdx b/docs/core_docs/docs/integrations/vectorstores/clickhouse.mdx
@@ -0,0 +1,37 @@
+---
+sidebar_class_name: node-only
+---
+
+import CodeBlock from "@theme/CodeBlock";
+
+# ClickHouse
+
+:::tip Compatibility
+Only available on Node.js.
+:::
+
+[ClickHouse](https://clickhouse.com/) is a robust and open-source columnar database that is used for handling analytical queries and efficient storage, ClickHouse is designed to provide a powerful combination of vector search and analytics.
+
+## Setup
+
+1. Launch a ClickHouse cluster. Refer to the [ClickHouse Installation Guide](https://clickhouse.com/docs/en/getting-started/install/) for details.
+2. After launching a ClickHouse cluster, retrieve the `Connection Details` from the cluster's `Actions` menu. You will need the host, port, username, and password.
+3. Install the required Node.js peer dependency for ClickHouse in your workspace.
+
+You will need to install the following peer dependencies:
+
+```bash npm2yarn
+npm install -S @clickhouse/client mysql2
+```
+
+## Index and Query Docs
+
+import InsertExample from "@examples/indexes/vector_stores/clickhouse_fromTexts.ts";
+
+<CodeBlock language="typescript">{InsertExample}</CodeBlock>
+
+## Query Docs From an Existing Collection
+
+import SearchExample from "@examples/indexes/vector_stores/clickhouse_search.ts";
+
+<CodeBlock language="typescript">{SearchExample}</CodeBlock>
diff --git a/docs/core_docs/docs/modules/data_connection/vectorstores/index.mdx b/docs/core_docs/docs/modules/data_connection/vectorstores/index.mdx
@@ -136,3 +136,4 @@ Here's a quick guide to help you pick the right vector store for your use case:
 - If you are looking for an online MPP (Massively Parallel Processing) data warehousing service, you might want to consider the [AnalyticDB](/docs/integrations/vectorstores/analyticdb) vector store.
 - If you're in search of a cost-effective vector database that allows run vector search with SQL, look no further than [MyScale](/docs/integrations/vectorstores/myscale).
 - If you're in search of a vector database that you can load from both the browser and server side, check out [CloseVector](/docs/integrations/vectorstores/closevector). It's a vector database that aims to be cross-platform.
+- If you're looking for a scalable, open-source columnar database with excellent performance for analytical queries, then consider [ClickHouse](/docs/integrations/vectorstores/clickhouse).
diff --git a/examples/.env.example b/examples/.env.example
@@ -38,6 +38,10 @@ MYSCALE_HOST=ADD_YOURS_HERE
 MYSCALE_PORT=ADD_YOURS_HERE
 MYSCALE_USERNAME=ADD_YOURS_HERE
 MYSCALE_PASSWORD=ADD_YOURS_HERE
+CLICKHOUSE_HOST=ADD_YOURS_HERE
+CLICKHOUSE_PORT=ADD_YOURS_HERE
+CLICKHOUSE_USERNAME=ADD_YOURS_HERE
+CLICKHOUSE_PASSWORD=ADD_YOURS_HERE
 REDIS_URL=ADD_YOURS_HERE
 SINGLESTORE_HOST=ADD_YOURS_HERE
 SINGLESTORE_PORT=ADD_YOURS_HERE

diff --git a/examples/package.json b/examples/package.json
@@ -22,7 +22,7 @@
   "author": "LangChain",
   "license": "MIT",
   "dependencies": {
-    "@clickhouse/client": "^0.0.14",
+    "@clickhouse/client": "^0.2.5",
     "@elastic/elasticsearch": "^8.4.0",
     "@getmetal/metal-sdk": "^4.0.0",
     "@getzep/zep-js": "^0.9.0",

diff --git a/examples/src/indexes/vector_stores/clickhouse_fromTexts.ts b/examples/src/indexes/vector_stores/clickhouse_fromTexts.ts
@@ -0,0 +1,35 @@
+import { ClickHouseStore } from "langchain/vectorstores/clickhouse";
+import { OpenAIEmbeddings } from "langchain/embeddings/openai";
+
+// Initialize ClickHouse store from texts
+const vectorStore = await ClickHouseStore.fromTexts(
+  ["Hello world", "Bye bye", "hello nice world"],
+  [
+    { id: 2, name: "2" },
+    { id: 1, name: "1" },
+    { id: 3, name: "3" },
+  ],
+  new OpenAIEmbeddings(),
+  {
+    host: process.env.CLICKHOUSE_HOST || "localhost",
+    port: process.env.CLICKHOUSE_PORT || 8443,
+    username: process.env.CLICKHOUSE_USER || "username",
+    password: process.env.CLICKHOUSE_PASSWORD || "password",
+    database: process.env.CLICKHOUSE_DATABASE || "default",
+    table: process.env.CLICKHOUSE_TABLE || "vector_table",
+  }
+);
+
+// Sleep 1 second to ensure that the search occurs after the successful insertion of data.
+// eslint-disable-next-line no-promise-executor-return
+await new Promise((resolve) => setTimeout(resolve, 1000));
+
+// Perform similarity search without filtering
+const results = await vectorStore.similaritySearch("hello world", 1);
+console.log(results);
+
+// Perform similarity search with filtering
+const filteredResults = await vectorStore.similaritySearch("hello world", 1, {
+  whereStr: "metadata.name = '1'",
+});
+console.log(filteredResults);
diff --git a/examples/src/indexes/vector_stores/clickhouse_search.ts b/examples/src/indexes/vector_stores/clickhouse_search.ts
@@ -0,0 +1,29 @@
+import { ClickHouseStore } from "langchain/vectorstores/clickhouse";
+import { OpenAIEmbeddings } from "langchain/embeddings/openai";
+
+// Initialize ClickHouse store
+const vectorStore = await ClickHouseStore.fromExistingIndex(
+  new OpenAIEmbeddings(),
+  {
+    host: process.env.CLICKHOUSE_HOST || "localhost",
+    port: process.env.CLICKHOUSE_PORT || 8443,
+    username: process.env.CLICKHOUSE_USER || "username",
+    password: process.env.CLICKHOUSE_PASSWORD || "password",
+    database: process.env.CLICKHOUSE_DATABASE || "default",
+    table: process.env.CLICKHOUSE_TABLE || "vector_table",
+  }
+);
+
+// Sleep 1 second to ensure that the search occurs after the successful insertion of data.
+// eslint-disable-next-line no-promise-executor-return
+await new Promise((resolve) => setTimeout(resolve, 1000));
+
+// Perform similarity search without filtering
+const results = await vectorStore.similaritySearch("hello world", 1);
+console.log(results);
+
+// Perform similarity search with filtering
+const filteredResults = await vectorStore.similaritySearch("hello world", 1, {
+  whereStr: "metadata.name = '1'",
+});
+console.log(filteredResults);
diff --git a/langchain/.env.example b/langchain/.env.example
@@ -38,6 +38,10 @@ MYSCALE_HOST=ADD_YOURS_HERE
 MYSCALE_PORT=ADD_YOURS_HERE
 MYSCALE_USERNAME=ADD_YOURS_HERE
 MYSCALE_PASSWORD=ADD_YOURS_HERE
+CLICKHOUSE_HOST=ADD_YOURS_HERE
+CLICKHOUSE_PORT=ADD_YOURS_HERE
+CLICKHOUSE_USERNAME=ADD_YOURS_HERE
+CLICKHOUSE_PASSWORD=ADD_YOURS_HERE
 FIGMA_ACCESS_TOKEN=ADD_YOURS_HERE
 REDIS_URL=ADD_YOURS_HERE
 ROCKSET_API_KEY=ADD_YOURS_HERE

diff --git a/langchain/.gitignore b/langchain/.gitignore
@@ -217,6 +217,9 @@ prompts.d.ts
 prompts/load.cjs
 prompts/load.js
 prompts/load.d.ts
+vectorstores/clickhouse.cjs
+vectorstores/clickhouse.js
+vectorstores/clickhouse.d.ts
 vectorstores/analyticdb.cjs
 vectorstores/analyticdb.js
 vectorstores/analyticdb.d.ts

diff --git a/langchain/package.json b/langchain/package.json
@@ -229,6 +229,9 @@
     "prompts/load.cjs",
     "prompts/load.js",
     "prompts/load.d.ts",
+    "vectorstores/clickhouse.cjs",
+    "vectorstores/clickhouse.js",
+    "vectorstores/clickhouse.d.ts",
     "vectorstores/analyticdb.cjs",
     "vectorstores/analyticdb.js",
     "vectorstores/analyticdb.d.ts",
@@ -858,7 +861,7 @@
     "@aws-sdk/credential-provider-node": "^3.388.0",
     "@aws-sdk/types": "^3.357.0",
     "@azure/storage-blob": "^12.15.0",
-    "@clickhouse/client": "^0.0.14",
+    "@clickhouse/client": "^0.2.5",
     "@cloudflare/ai": "^1.0.12",
     "@cloudflare/workers-types": "^4.20230922.0",
     "@elastic/elasticsearch": "^8.4.0",
@@ -1000,7 +1003,7 @@
     "@aws-sdk/client-sfn": "^3.310.0",
     "@aws-sdk/credential-provider-node": "^3.388.0",
     "@azure/storage-blob": "^12.15.0",
-    "@clickhouse/client": "^0.0.14",
+    "@clickhouse/client": "^0.2.5",
     "@cloudflare/ai": "^1.0.12",
     "@elastic/elasticsearch": "^8.4.0",
     "@getmetal/metal-sdk": "*",
@@ -1796,6 +1799,11 @@
       "import": "./prompts/load.js",
       "require": "./prompts/load.cjs"
     },
+    "./vectorstores/clickhouse": {
+      "types": "./vectorstores/clickhouse.d.ts",
+      "import": "./vectorstores/clickhouse.js",
+      "require": "./vectorstores/clickhouse.cjs"
+    },
     "./vectorstores/analyticdb": {
       "types": "./vectorstores/analyticdb.d.ts",
       "import": "./vectorstores/analyticdb.js",

diff --git a/langchain/scripts/create-entrypoints.js b/langchain/scripts/create-entrypoints.js
@@ -90,6 +90,7 @@ const entrypoints = {
   prompts: "prompts/index",
   "prompts/load": "prompts/load",
   // vectorstores
+  "vectorstores/clickhouse": "vectorstores/clickhouse",
   "vectorstores/analyticdb": "vectorstores/analyticdb",
   "vectorstores/base": "vectorstores/base",
   "vectorstores/cassandra": "vectorstores/cassandra",
@@ -376,6 +377,7 @@ const requiresOptionalDependency = [
   "prompts/load",
   "vectorstores/analyticdb",
   "vectorstores/cassandra",
+  "vectorstores/clickhouse",
   "vectorstores/chroma",
   "vectorstores/cloudflare_vectorize",
   "vectorstores/closevector/web",

diff --git a/langchain/src/load/import_constants.ts b/langchain/src/load/import_constants.ts
@@ -39,6 +39,7 @@ export const optionalImportEntrypoints = [
   "langchain/llms/writer",
   "langchain/llms/portkey",
   "langchain/prompts/load",
+  "langchain/vectorstores/clickhouse",
   "langchain/vectorstores/analyticdb",
   "langchain/vectorstores/cassandra",
   "langchain/vectorstores/convex",

diff --git a/langchain/src/load/import_type.d.ts b/langchain/src/load/import_type.d.ts
@@ -115,6 +115,9 @@ export interface OptionalImportMap {
   "langchain/prompts/load"?:
     | typeof import("../prompts/load.js")
     | Promise<typeof import("../prompts/load.js")>;
+  "langchain/vectorstores/clickhouse"?:
+    | typeof import("../vectorstores/clickhouse.js")
+    | Promise<typeof import("../vectorstores/clickhouse.js")>;
   "langchain/vectorstores/analyticdb"?:
     | typeof import("../vectorstores/analyticdb.js")
     | Promise<typeof import("../vectorstores/analyticdb.js")>;