Feat: remove geo files from repo & refine tests (v2fly#869)

.github/workflows/release.yml

@@ -118,6 +118,11 @@ jobs:
           mv v2ray v2ray.exe
           mv v2ctl v2ctl.exe
 
+      - name: Download geo files
+        run: |
+          wget -O release/config/geoip.dat "https://raw.githubusercontent.com/v2fly/geoip/release/geoip.dat"
+          wget -O release/config/geosite.dat "https://raw.githubusercontent.com/v2fly/domain-list-community/release/dlc.dat"
+
       - name: Prepare package
         run: cp -v ./release/config/*.* ./build_assets
 

.gitignore

@@ -25,4 +25,4 @@ vprotogen
 !infra/vprotogen/
 errorgen
 !common/errors/errorgen/
-infra/conf/*.dat
+*.dat

app/router/condition_geoip_test.go

@@ -1,11 +1,13 @@
 package router_test
 
 import (
+	"errors"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
-	"github.com/golang/protobuf/proto"
+	"google.golang.org/protobuf/proto"
 
 	"github.com/v2fly/v2ray-core/v4/app/router"
 	"github.com/v2fly/v2ray-core/v4/common"
@@ -18,11 +20,18 @@ func init() {
 	wd, err := os.Getwd()
 	common.Must(err)
 
-	if _, err := os.Stat(platform.GetAssetLocation("geoip.dat")); err != nil && os.IsNotExist(err) {
-		common.Must(filesystem.CopyFile(platform.GetAssetLocation("geoip.dat"), filepath.Join(wd, "..", "..", "release", "config", "geoip.dat")))
-	}
-	if _, err := os.Stat(platform.GetAssetLocation("geosite.dat")); err != nil && os.IsNotExist(err) {
-		common.Must(filesystem.CopyFile(platform.GetAssetLocation("geosite.dat"), filepath.Join(wd, "..", "..", "release", "config", "geosite.dat")))
+	tempPath := filepath.Join(wd, "..", "..", "testing", "temp")
+	geoipPath := filepath.Join(tempPath, "geoip.dat")
+
+	os.Setenv("v2ray.location.asset", tempPath)
+
+	if _, err := os.Stat(platform.GetAssetLocation("geoip.dat")); err != nil && errors.Is(err, os.ErrNotExist) {
+		if _, err := os.Stat(geoipPath); err != nil && errors.Is(err, os.ErrNotExist) {
+			common.Must(os.MkdirAll(tempPath, 0755))
+			geoipBytes, err := common.FetchHTTPContent(geoipURL)
+			common.Must(err)
+			common.Must(filesystem.WriteFile(geoipPath, geoipBytes))
+		}
 	}
 }
 
@@ -195,7 +204,7 @@ func loadGeoIP(country string) ([]*router.CIDR, error) {
 	}
 
 	for _, geoip := range geoipList.Entry {
-		if geoip.CountryCode == country {
+		if strings.EqualFold(geoip.CountryCode, country) {
 			return geoip.Cidr, nil
 		}
 	}

app/router/condition_test.go

@@ -1,16 +1,17 @@
 package router_test
 
 import (
+	"errors"
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"testing"
 
-	"github.com/golang/protobuf/proto"
+	"google.golang.org/protobuf/proto"
 
-	. "github.com/v2fly/v2ray-core/v4/app/router"
+	"github.com/v2fly/v2ray-core/v4/app/router"
 	"github.com/v2fly/v2ray-core/v4/common"
-	"github.com/v2fly/v2ray-core/v4/common/errors"
 	"github.com/v2fly/v2ray-core/v4/common/net"
 	"github.com/v2fly/v2ray-core/v4/common/platform"
 	"github.com/v2fly/v2ray-core/v4/common/platform/filesystem"
@@ -25,11 +26,28 @@ func init() {
 	wd, err := os.Getwd()
 	common.Must(err)
 
-	if _, err := os.Stat(platform.GetAssetLocation("geoip.dat")); err != nil && os.IsNotExist(err) {
-		common.Must(filesystem.CopyFile(platform.GetAssetLocation("geoip.dat"), filepath.Join(wd, "..", "..", "release", "config", "geoip.dat")))
+	tempPath := filepath.Join(wd, "..", "..", "testing", "temp")
+	geoipPath := filepath.Join(tempPath, "geoip.dat")
+	geositePath := filepath.Join(tempPath, "geosite.dat")
+
+	os.Setenv("v2ray.location.asset", tempPath)
+
+	if _, err := os.Stat(platform.GetAssetLocation("geoip.dat")); err != nil && errors.Is(err, os.ErrNotExist) {
+		if _, err := os.Stat(geoipPath); err != nil && errors.Is(err, os.ErrNotExist) {
+			common.Must(os.MkdirAll(tempPath, 0755))
+			geoipBytes, err := common.FetchHTTPContent(geoipURL)
+			common.Must(err)
+			common.Must(filesystem.WriteFile(geoipPath, geoipBytes))
+		}
 	}
-	if _, err := os.Stat(platform.GetAssetLocation("geosite.dat")); err != nil && os.IsNotExist(err) {
-		common.Must(filesystem.CopyFile(platform.GetAssetLocation("geosite.dat"), filepath.Join(wd, "..", "..", "release", "config", "geosite.dat")))
+
+	if _, err := os.Stat(platform.GetAssetLocation("geosite.dat")); err != nil && errors.Is(err, os.ErrNotExist) {
+		if _, err := os.Stat(geositePath); err != nil && errors.Is(err, os.ErrNotExist) {
+			common.Must(os.MkdirAll(tempPath, 0755))
+			geositeBytes, err := common.FetchHTTPContent(geositeURL)
+			common.Must(err)
+			common.Must(filesystem.WriteFile(geositePath, geositeBytes))
+		}
 	}
 }
 
@@ -56,23 +74,23 @@ func TestRoutingRule(t *testing.T) {
 	}
 
 	cases := []struct {
-		rule *RoutingRule
+		rule *router.RoutingRule
 		test []ruleTest
 	}{
 		{
-			rule: &RoutingRule{
-				Domain: []*Domain{
+			rule: &router.RoutingRule{
+				Domain: []*router.Domain{
 					{
 						Value: "v2fly.org",
-						Type:  Domain_Plain,
+						Type:  router.Domain_Plain,
 					},
 					{
 						Value: "google.com",
-						Type:  Domain_Domain,
+						Type:  router.Domain_Domain,
 					},
 					{
 						Value: "^facebook\\.com$",
-						Type:  Domain_Regex,
+						Type:  router.Domain_Regex,
 					},
 				},
 			},
@@ -108,8 +126,8 @@ func TestRoutingRule(t *testing.T) {
 			},
 		},
 		{
-			rule: &RoutingRule{
-				Cidr: []*CIDR{
+			rule: &router.RoutingRule{
+				Cidr: []*router.CIDR{
 					{
 						Ip:     []byte{8, 8, 8, 8},
 						Prefix: 32,
@@ -144,10 +162,10 @@ func TestRoutingRule(t *testing.T) {
 			},
 		},
 		{
-			rule: &RoutingRule{
-				Geoip: []*GeoIP{
+			rule: &router.RoutingRule{
+				Geoip: []*router.GeoIP{
 					{
-						Cidr: []*CIDR{
+						Cidr: []*router.CIDR{
 							{
 								Ip:     []byte{8, 8, 8, 8},
 								Prefix: 32,
@@ -184,8 +202,8 @@ func TestRoutingRule(t *testing.T) {
 			},
 		},
 		{
-			rule: &RoutingRule{
-				SourceCidr: []*CIDR{
+			rule: &router.RoutingRule{
+				SourceCidr: []*router.CIDR{
 					{
 						Ip:     []byte{192, 168, 0, 0},
 						Prefix: 16,
@@ -204,7 +222,7 @@ func TestRoutingRule(t *testing.T) {
 			},
 		},
 		{
-			rule: &RoutingRule{
+			rule: &router.RoutingRule{
 				UserEmail: []string{
 					"[email protected]",
 				},
@@ -225,7 +243,7 @@ func TestRoutingRule(t *testing.T) {
 			},
 		},
 		{
-			rule: &RoutingRule{
+			rule: &router.RoutingRule{
 				Protocol: []string{"http"},
 			},
 			test: []ruleTest{
@@ -236,7 +254,7 @@ func TestRoutingRule(t *testing.T) {
 			},
 		},
 		{
-			rule: &RoutingRule{
+			rule: &router.RoutingRule{
 				InboundTag: []string{"test", "test1"},
 			},
 			test: []ruleTest{
@@ -251,7 +269,7 @@ func TestRoutingRule(t *testing.T) {
 			},
 		},
 		{
-			rule: &RoutingRule{
+			rule: &router.RoutingRule{
 				PortList: &net.PortList{
 					Range: []*net.PortRange{
 						{From: 443, To: 443},
@@ -279,7 +297,7 @@ func TestRoutingRule(t *testing.T) {
 			},
 		},
 		{
-			rule: &RoutingRule{
+			rule: &router.RoutingRule{
 				SourcePortList: &net.PortList{
 					Range: []*net.PortRange{
 						{From: 123, To: 123},
@@ -307,7 +325,7 @@ func TestRoutingRule(t *testing.T) {
 			},
 		},
 		{
-			rule: &RoutingRule{
+			rule: &router.RoutingRule{
 				Protocol:   []string{"http"},
 				Attributes: "attrs[':path'].startswith('/test')",
 			},
@@ -333,32 +351,31 @@ func TestRoutingRule(t *testing.T) {
 	}
 }
 
-func loadGeoSite(country string) ([]*Domain, error) {
+func loadGeoSite(country string) ([]*router.Domain, error) {
 	geositeBytes, err := filesystem.ReadAsset("geosite.dat")
 	if err != nil {
 		return nil, err
 	}
-	var geositeList GeoSiteList
+	var geositeList router.GeoSiteList
 	if err := proto.Unmarshal(geositeBytes, &geositeList); err != nil {
 		return nil, err
 	}
 
 	for _, site := range geositeList.Entry {
-		if site.CountryCode == country {
+		if strings.EqualFold(site.CountryCode, country) {
 			return site.Domain, nil
 		}
 	}
 
 	return nil, errors.New("country not found: " + country)
 }
-
 func TestChinaSites(t *testing.T) {
 	domains, err := loadGeoSite("CN")
 	common.Must(err)
 
-	matcher, err := NewDomainMatcher(domains)
+	matcher, err := router.NewDomainMatcher(domains)
 	common.Must(err)
-	acMatcher, err := NewMphMatcherGroup(domains)
+	acMatcher, err := router.NewMphMatcherGroup(domains)
 	common.Must(err)
 
 	type TestCase struct {
@@ -403,7 +420,7 @@ func BenchmarkMphDomainMatcher(b *testing.B) {
 	domains, err := loadGeoSite("CN")
 	common.Must(err)
 
-	matcher, err := NewMphMatcherGroup(domains)
+	matcher, err := router.NewMphMatcherGroup(domains)
 	common.Must(err)
 
 	type TestCase struct {
@@ -445,7 +462,7 @@ func BenchmarkDomainMatcher(b *testing.B) {
 	domains, err := loadGeoSite("CN")
 	common.Must(err)
 
-	matcher, err := NewDomainMatcher(domains)
+	matcher, err := router.NewDomainMatcher(domains)
 	common.Must(err)
 
 	type TestCase struct {
@@ -484,12 +501,12 @@ func BenchmarkDomainMatcher(b *testing.B) {
 }
 
 func BenchmarkMultiGeoIPMatcher(b *testing.B) {
-	var geoips []*GeoIP
+	var geoips []*router.GeoIP
 
 	{
 		ips, err := loadGeoIP("CN")
 		common.Must(err)
-		geoips = append(geoips, &GeoIP{
+		geoips = append(geoips, &router.GeoIP{
 			CountryCode: "CN",
 			Cidr:        ips,
 		})
@@ -498,7 +515,7 @@ func BenchmarkMultiGeoIPMatcher(b *testing.B) {
 	{
 		ips, err := loadGeoIP("JP")
 		common.Must(err)
-		geoips = append(geoips, &GeoIP{
+		geoips = append(geoips, &router.GeoIP{
 			CountryCode: "JP",
 			Cidr:        ips,
 		})
@@ -507,7 +524,7 @@ func BenchmarkMultiGeoIPMatcher(b *testing.B) {
 	{
 		ips, err := loadGeoIP("CA")
 		common.Must(err)
-		geoips = append(geoips, &GeoIP{
+		geoips = append(geoips, &router.GeoIP{
 			CountryCode: "CA",
 			Cidr:        ips,
 		})
@@ -516,13 +533,13 @@ func BenchmarkMultiGeoIPMatcher(b *testing.B) {
 	{
 		ips, err := loadGeoIP("US")
 		common.Must(err)
-		geoips = append(geoips, &GeoIP{
+		geoips = append(geoips, &router.GeoIP{
 			CountryCode: "US",
 			Cidr:        ips,
 		})
 	}
 
-	matcher, err := NewMultiGeoIPMatcher(geoips, false)
+	matcher, err := router.NewMultiGeoIPMatcher(geoips, false)
 	common.Must(err)
 
 	ctx := withOutbound(&session.Outbound{Target: net.TCPDestination(net.ParseAddress("8.8.8.8"), 80)})

app/router/constant_test.go

@@ -0,0 +1,6 @@
+package router_test
+
+const (
+	geoipURL   = "https://raw.githubusercontent.com/v2fly/geoip/release/geoip.dat"
+	geositeURL = "https://raw.githubusercontent.com/v2fly/domain-list-community/release/dlc.dat"
+)