feat: support ModelScope for model registry downloading (TabbyML#477)

* feat: update cache info file after each file got downloaded

* refactor: extract Downloader for model downloading logic

* refactor: extract HuggingFaceRegistry

* refactor: extract serde_json to workspace dependency

* feat: add ModelScopeRegistry

* refactor: extract registry to its sub dir.

* feat: add scripts to mirror hf model to modelscope

Cargo.toml

@@ -22,6 +22,7 @@ homepage = "https://github.com/TabbyML/tabby"
 [workspace.dependencies]
 lazy_static = "1.4.0"
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1"
 serdeconv = "0.4.1"
 tokio = "1.28"
 tokio-util = "0.7"

crates/http-api-bindings/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 async-trait.workspace = true
 reqwest = { workspace = true, features = ["json"] }
 serde = { workspace = true, features = ["derive"] }
-serde_json = "1.0.105"
+serde_json = { workspace = true }
 tabby-inference = { version = "0.1.0", path = "../tabby-inference" }
 
 [dev-dependencies]

crates/tabby-download/Cargo.toml

@@ -13,3 +13,7 @@ serde = { workspace = true }
 serdeconv = { workspace = true }
 tracing = { workspace = true }
 tokio-retry = "0.3.0"
+urlencoding = "2.1.3"
+serde_json = { workspace = true }
+cached = { version = "0.46.0", features = ["async", "proc_macro"] }
+async-trait = { workspace = true }

crates/tabby-download/src/cache_info.rs

@@ -1,6 +1,6 @@
 use std::{collections::HashMap, fs, path::Path};
 
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 use serde::{Deserialize, Serialize};
 use tabby_common::path::ModelDir;
 
@@ -33,15 +33,6 @@ impl CacheInfo {
         self.etags.get(path).map(|x| x.as_str())
     }
 
-    pub fn remote_cache_key(res: &reqwest::Response) -> Result<&str> {
-        let key = res
-            .headers()
-            .get("etag")
-            .ok_or(anyhow!("etag key missing"))?
-            .to_str()?;
-        Ok(key)
-    }
-
     pub async fn set_local_cache_key(&mut self, path: &str, cache_key: &str) {
         self.etags.insert(path.to_string(), cache_key.to_string());
     }