add the function of dynamically loading IP dictionary information

conf/ad_server.toml

@@ -1,6 +1,7 @@
 # ip字典
 GeoBlockFileName = "./data/GeoLiteCity-Blocks.csv"
 GeoLocationFileName = "./data/GeoLiteCity-Location.csv"
+IpFileReloadInterval = 1
 
 # 广告字典
 AdFileName = "./data/ad_info.txt"
@@ -19,3 +20,6 @@ ConversionLogFileName = "./log/ad_conversion.log"
 ImpressionTrackUrlPrefix = "http://localhost:8001/ad/impression"
 ClickTrackUrlPrefix = "http://localhost:8001/ad/click"
 ConversionTrackUrlPrefix = "http://localhost:8001/ad/conversion"
+
+#adserver config
+AdServerPort = 8001

src/adhandler/search_handler.go

@@ -23,11 +23,9 @@ func SearchHandler(w http.ResponseWriter, r *http.Request) {
 		req.SlotId = uint32(slotId)
 	}
 	// ad_num
-	var reqAdNum int
 	if len(r.Form["ad_num"]) > 0 {
-		adNum, _ := strconv.ParseUint(r.Form["ad_num"][0], 10, 32)
-		reqAdNum, _ = strconv.Atoi(r.Form["ad_num"][0])
-		req.AdNum = uint32(adNum)
+		reqAdNum, _ := strconv.ParseUint(r.Form["ad_num"][0], 10, 32)
+		req.ReqAdNum = uint32(reqAdNum)
 	}
 	// ip
 	if len(r.Form["ip"]) > 0 {
@@ -54,7 +52,8 @@ func SearchHandler(w http.ResponseWriter, r *http.Request) {
 	// search by request ip
 	var unitIdList1 []uint32
 	var exist1 bool
-	locationInfo := adserver.SearchLocationByIp(req.Ip)
+	ipDataInfo := adserver.LocationDict.GetCurrentIpData()
+	locationInfo := ipDataInfo.SearchLocationByIp(req.Ip)
 	if locationInfo != nil {
 		country := locationInfo.Country
 		city := locationInfo.City
@@ -86,16 +85,18 @@ func SearchHandler(w http.ResponseWriter, r *http.Request) {
 	var res = &adserver.Response{}
 	adList := make([]adserver.AdInfo, 0, 1)
 	unitIdMap := make(map[int] bool)
-	var unitIdsStr, creativeIdsStr string 
-	if unitIdList != nil && reqAdNum >= 1 {
+	var unitIdsStr, creativeIdsStr string
+	resAdNum := 0 
+	if unitIdList != nil && req.ReqAdNum >= 1 {
 		unitNum = len(unitIdList)
 		random := rand.New(rand.NewSource(time.Now().UnixNano()))
-		for i := 0; i < unitNum && i < reqAdNum; i++ {
+		for i := 0; i < unitNum && i < int(req.ReqAdNum); i++ {
 			randIndex := random.Intn(unitNum)
 			if unitIdMap[randIndex] {
 				i--
 				continue
 			}
+			resAdNum++
 			unitIdMap[randIndex] = true
 			unitId := unitIdList[randIndex]
 			unitInfo := adData.AdUnitMap[unitId]
@@ -114,7 +115,7 @@ func SearchHandler(w http.ResponseWriter, r *http.Request) {
 			adInfo.ClickTrackUrl = buildClickTrackUrl(req, adInfo)
 			adInfo.ConversionTrackUrl = buildConversionTrackUrl(req, adInfo)
 			adList = append(adList, adInfo)
-			if i == unitNum - 1 || i == reqAdNum - 1 {
+			if i == unitNum - 1 || i == int(req.ReqAdNum) - 1 {
 				unitIdsStr += fmt.Sprint(adInfo.UnitId)
 				creativeIdsStr += fmt.Sprint(adInfo.CreativeId)
 			} else {
@@ -130,9 +131,9 @@ func SearchHandler(w http.ResponseWriter, r *http.Request) {
 	}
     adserver.SearchLog.Info(fmt.Sprintf(
 			"searchId=%s slotId=%d adNum=%d iP=%s deviceId=%s oS=%d osVersion=%s " +
-			"unitId=%s creativeId=%s\n",
-			req.SearchId, req.SlotId, req.AdNum, req.Ip, req.DeviceId, req.Os, req.OsVersion,
-			unitIdsStr, creativeIdsStr))
+			"unitId=%s creativeId=%s resAdNum=%d\n",
+			req.SearchId, req.SlotId, req.ReqAdNum, req.Ip, req.DeviceId, req.Os, req.OsVersion,
+			unitIdsStr, creativeIdsStr,resAdNum))
 
 	resBytes, _ := json.Marshal(res)
 	w.Write(resBytes)

src/adserver/global_conf.go

@@ -8,6 +8,7 @@ import (
 type GlobalConf struct {
 	GeoBlockFileName string
 	GeoLocationFileName string
+	IpFileReloadInterval int64
 	AdFileName string
 	AdFileReloadInterval int64
 	// log config
@@ -21,6 +22,7 @@ type GlobalConf struct {
 	ImpressionTrackUrlPrefix string
 	ClickTrackUrlPrefix string
 	ConversionTrackUrlPrefix string
+	AdServerPort int
 }
 
 var GlobalConfObject *GlobalConf
@@ -54,4 +56,5 @@ func LoadGlobalConf(configPath, configFileName string)  {
 	fmt.Printf("ImpressionTrackUrlPrefix=%s\n", GlobalConfObject.ImpressionTrackUrlPrefix)
 	fmt.Printf("ClickTrackUrlPrefix=%s\n", GlobalConfObject.ClickTrackUrlPrefix)
 	fmt.Printf("ConversionTrackUrlPrefix=%s\n", GlobalConfObject.ConversionTrackUrlPrefix)
+	fmt.Printf("AdServerPort=%d\n", GlobalConfObject.AdServerPort)
 }

src/adserver/location_dict.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"sort"
 	"utils"
+	"time"
 )
 
 type GeoLocationInfo struct {
@@ -24,28 +25,59 @@ type LocationInfo struct {
 	City string
 }
 
-type IpDict struct {
+type IpDataInfo struct {
 	ipPairs utils.IpPairs
 	ipLocationMap map[utils.IpPair]*LocationInfo
 }
 
+type IpDict struct {
+	IpDataArray []*IpDataInfo
+	CurrentIndex uint32
+	BlockLastModifiedTime int64
+	LocationLastModifiedTime int64
+}
+
 var LocationDict *IpDict
 
 func init() {
 	LocationDict = &IpDict{
+		IpDataArray: make([]*IpDataInfo, 2, 2),
+		CurrentIndex: 0,
+		BlockLastModifiedTime: 0,
+		LocationLastModifiedTime: 0,
+	}
+	for i := 0; i < 2; i++ {
+		LocationDict.IpDataArray[i] = NewIpDataInfo()
+	}
+}
+
+// 初始化之后首次加载Ip字典信息
+func (ipDict *IpDict) Load() {
+	ipDataInfo := LoadLocationDict(GlobalConfObject.GeoBlockFileName,
+		GlobalConfObject.GeoLocationFileName)
+	ipDict.IpDataArray[ipDict.CurrentIndex] = ipDataInfo
+	blockFileStat, _ := os.Stat(GlobalConfObject.GeoBlockFileName)
+    locationFileStat, _ := os.Stat(GlobalConfObject.GeoLocationFileName)
+	ipDict.BlockLastModifiedTime = blockFileStat.ModTime().Unix()
+	ipDict.LocationLastModifiedTime = locationFileStat.ModTime().Unix()
+}
+
+func NewIpDataInfo() *IpDataInfo {
+	return &IpDataInfo{
 		ipPairs: make(utils.IpPairs, 0),
 		ipLocationMap: make(map[utils.IpPair]*LocationInfo),
 	}
 }
 
-func LoadLocationDict(blockFileName, locationFileName string) {
+func LoadLocationDict(blockFileName, locationFileName string) *IpDataInfo {
 	dictFile, err := os.Open(blockFileName)
 	if err != nil {
 		AdServerLog.Error(fmt.Sprintf("open file error, name=%s\n", blockFileName))
 		panic(-1)
 	}
 	defer dictFile.Close()
 
+	ipDataInfo := NewIpDataInfo()
 	geoLocationMap := loadGeoLocation(locationFileName)
 	br := bufio.NewReader(dictFile)
 	for {
@@ -85,21 +117,55 @@ func LoadLocationDict(blockFileName, locationFileName string) {
 			BeginIp: beginIp,
 			EndIp: endIp,
 		}
-		LocationDict.ipPairs = append(LocationDict.ipPairs, ipPair)
-		LocationDict.ipLocationMap[ipPair] = locationInfo
+		ipDataInfo.ipPairs = append(ipDataInfo.ipPairs, ipPair)
+		ipDataInfo.ipLocationMap[ipPair] = locationInfo
 	}
 
-	sort.Sort(LocationDict.ipPairs)
+	sort.Sort(ipDataInfo.ipPairs)
 	AdServerLog.Info(fmt.Sprintf(
 		"read dict success, blockFileName=%s locationFileName=%s\n",
 		blockFileName, locationFileName))
 	AdServerLog.Info(fmt.Sprintf(
-		"location dict size=%d\n", len(LocationDict.ipPairs)))
+		"location dict size=%d\n", len(ipDataInfo.ipPairs)))
+
+	return ipDataInfo
+}
+
+// 启动定时器，用于定期重新加载Ip字典信息
+func (locationDict *IpDict) StartReloadTimer() {
+	duration := int64(time.Second) * GlobalConfObject.IpFileReloadInterval
+	t := time.NewTicker(time.Duration(duration))
+	go func() {
+		for t1 := range t.C {
+			AdServerLog.Debug("IpDict reload timer execute")
+			blockFileStat, _ := os.Stat(GlobalConfObject.GeoBlockFileName)
+			locationFileStat, _ := os.Stat(GlobalConfObject.GeoLocationFileName)
+			blockCurrentModifiedTime := blockFileStat.ModTime().Unix()
+			locationCurrentModifiedTime := locationFileStat.ModTime().Unix()
+			// 如果文件有更新，则重新加载广告内容
+			if blockCurrentModifiedTime > locationDict.BlockLastModifiedTime || locationCurrentModifiedTime > locationDict.LocationLastModifiedTime {
+				AdServerLog.Info(fmt.Sprintf("start reload ad info dict at %s",
+					t1.Format("2006-01-02 03:04:05")))
+				LoadLocationDict(
+					GlobalConfObject.GeoBlockFileName,
+					GlobalConfObject.GeoLocationFileName)
+				nextIndex := 1 - locationDict.CurrentIndex
+				locationDict.CurrentIndex = nextIndex
+				locationDict.BlockLastModifiedTime = blockCurrentModifiedTime
+				locationDict.LocationLastModifiedTime = locationCurrentModifiedTime
+			}
+		}
+	}()
+}
+
+// 获取当前可用的Ip字典信息
+func (ipDict *IpDict) GetCurrentIpData() *IpDataInfo {
+	return ipDict.IpDataArray[ipDict.CurrentIndex]
 }
 
-func SearchLocationByIp(ipString string) *LocationInfo {
+func (ipDataInfo *IpDataInfo) SearchLocationByIp(ipString string) *LocationInfo {
 	ip := utils.StringIpToUint(ipString)
-	ipPairs := LocationDict.ipPairs
+	ipPairs := ipDataInfo.ipPairs
 	size := len(ipPairs)
 	if size == 0 || ip < ipPairs[0].BeginIp || ip > ipPairs[size - 1].EndIp {
 		return nil
@@ -109,7 +175,7 @@ func SearchLocationByIp(ipString string) *LocationInfo {
 	for left <= right {
 		mid := (left + right) / 2
 		if ip >= ipPairs[mid].BeginIp && ip <= ipPairs[mid].EndIp {
-			return LocationDict.ipLocationMap[ipPairs[mid]];
+			return ipDataInfo.ipLocationMap[ipPairs[mid]];
 		} else if ip < ipPairs[mid].BeginIp {
 			right = mid - 1
 		} else if ip > ipPairs[mid].EndIp {

src/adserver/request.go

@@ -3,7 +3,7 @@ package adserver
 type Request struct {
 	SearchId string
 	SlotId uint32
-	AdNum uint32
+	ReqAdNum uint32
 	Ip string
 	DeviceId string
 	Os uint32 // 0:android, 1:ios

src/main/main.go

@@ -4,14 +4,16 @@ import (
 	"net/http"
 	"adserver"
 	"adhandler"
+	"strconv"
 )
 
 func main() {
 	adserver.LoadGlobalConf("./conf", "ad_server")
 	adserver.InitLog(adserver.GlobalConfObject)
-	adserver.LoadLocationDict(
-		adserver.GlobalConfObject.GeoBlockFileName,
-		adserver.GlobalConfObject.GeoLocationFileName)
+	//加载位置字典
+	adserver.LocationDict.Load()
+	adserver.LocationDict.StartReloadTimer()
+
 
 	// 初始化并加载广告信息
 	adserver.AdDictObject = adserver.NewAdDict(adserver.GlobalConfObject.AdFileName)
@@ -22,5 +24,6 @@ func main() {
 	http.HandleFunc("/ad/impression",adhandler.ImpressionHandler)
 	http.HandleFunc("/ad/click",adhandler.ClickHandler)
 	http.HandleFunc("/ad/conversion",adhandler.ConversionHandler)
-	http.ListenAndServe(":8001", nil)
+	listenPort := ":" + strconv.Itoa(adserver.GlobalConfObject.AdServerPort)
+	http.ListenAndServe(listenPort, nil)
 }