location dict

src/adhandler/search_handler.go

@@ -9,6 +9,7 @@ import (
 	"math/rand"
 	"adserver"
 	"strings"
+	"fmt"
 )
 
 func SearchHandler(w http.ResponseWriter, r *http.Request) {
@@ -47,16 +48,41 @@ func SearchHandler(w http.ResponseWriter, r *http.Request) {
 		req.OsVersion = r.Form["os_version"][0]
 	}
 
-	// TODO: read from location dict
-	country := "CN"
-	city := "ALL"
-	key := strings.ToLower(country) + "_" + strings.ToLower(city)
+	// search by request ip
 	adDict := adserver.AdDict
-	unitIdList, exist := adDict.LocationUnitMap[key]
+	var unitIdList1 []uint32
+	var exist1 bool
+	locationInfo := adserver.SearchLocationByIp(req.Ip)
+	if locationInfo != nil {
+		country := locationInfo.Country
+		city := locationInfo.City
+		fmt.Printf("ip=%s country=%s city=%s\n", req.Ip, country, city)
+		key := strings.ToLower(country) + "_" + strings.ToLower(city)
+		unitIdList1, exist1 = adDict.LocationUnitMap[key]
+	}
+	// search by CN_ALL
+	key := "cn_all"
+	unitIdList2, exist2 := adDict.LocationUnitMap[key]
+	// merge two unit id list
+	unitNum := 0
+	if exist1 {
+		unitNum += len(unitIdList1)
+	}
+	if exist2 {
+		unitNum += len(unitIdList2)
+	}
+	unitIdList := make([]uint32, unitNum)
+	if exist1 && unitIdList1 != nil {
+		copy(unitIdList, unitIdList1)
+	}
+	if exist2 && unitIdList2 != nil {
+		copy(unitIdList, unitIdList2)
+	}
 
+	// select one from unit id list
 	var res = &adserver.Response{}
-	if exist {
-		unitNum := len(unitIdList)
+	if unitIdList != nil {
+		unitNum = len(unitIdList)
 		random := rand.New(rand.NewSource(time.Now().UnixNano()))
 		randIndex := random.Intn(unitNum)
 		unitId := unitIdList[randIndex]

src/adserver/ad_dict.go

@@ -40,8 +40,7 @@ func init()  {
 	}
 }
 
-func ReadAdDict() {
-	dictFileName := "./data/ad_info.txt"
+func ReadAdDict(dictFileName string) {
 	dictFile, err := os.Open(dictFileName)
 	if err != nil {
 		fmt.Printf("open file error, name=%s\n", dictFileName)

src/adserver/location_dict.go

@@ -0,0 +1,154 @@
+package adserver
+
+import (
+	"io"
+	"os"
+	"fmt"
+	"strings"
+	"bufio"
+	"strconv"
+	"sort"
+	"utils"
+)
+
+type GeoLocationInfo struct {
+	LocId uint64
+	Country string
+	City string
+}
+
+type LocationInfo struct {
+	BeginIp uint32
+	EndIp uint32
+	Country string
+	City string
+}
+
+type IpDict struct {
+	ipPairs utils.IpPairs
+	ipLocationMap map[utils.IpPair]*LocationInfo
+}
+
+var LocationDict *IpDict
+
+func init() {
+	LocationDict = &IpDict{
+		ipPairs: make(utils.IpPairs, 0),
+		ipLocationMap: make(map[utils.IpPair]*LocationInfo),
+	}
+}
+
+func LoadLocationDict(blockFileName, locationFileName string) {
+	dictFile, err := os.Open(blockFileName)
+	if err != nil {
+		fmt.Printf("open file error, name=%s\n", blockFileName)
+	}
+	defer dictFile.Close()
+
+	geoLocationMap := loadGeoLocation(locationFileName)
+	br := bufio.NewReader(dictFile)
+	for {
+		line, _, err := br.ReadLine()
+		if err == io.EOF {
+			break
+		}
+		lineString := string(line)
+		if len(lineString) == 0 || lineString[0] == '#' {
+			continue
+		}
+		lines := strings.Split(lineString, ",")
+		if len(lines) != 3 {
+			fmt.Printf("invalid format, blockFileName=%s, line=%s\n",
+				blockFileName, lineString)
+			continue
+		}
+
+		tmpInt, _ := strconv.ParseUint(strings.Trim(lines[0], "\""), 10 ,32)
+		beginIp := uint32(tmpInt)
+		tmpInt, _ = strconv.ParseUint(strings.Trim(lines[1], "\""), 10 ,32)
+		endIp := uint32(tmpInt)
+		locId, _ := strconv.ParseUint(strings.Trim(lines[2], "\""), 10 ,64)
+		geoLocationInfo, exist := geoLocationMap[locId]
+		if !exist {
+			//fmt.Printf("geoLocationInfo not found, locId=%d\n", locId)
+			continue
+		}
+		locationInfo := &LocationInfo{
+			BeginIp: beginIp,
+			EndIp: endIp,
+			Country: geoLocationInfo.Country,
+			City: geoLocationInfo.City,
+		}
+		ipPair := utils.IpPair{
+			BeginIp: beginIp,
+			EndIp: endIp,
+		}
+		LocationDict.ipPairs = append(LocationDict.ipPairs, ipPair)
+		LocationDict.ipLocationMap[ipPair] = locationInfo
+	}
+
+	sort.Sort(LocationDict.ipPairs)
+	fmt.Printf("read dict success, blockFileName=%s locationFileName=%s\n", blockFileName, locationFileName)
+	fmt.Printf("location dict size=%d\n", len(LocationDict.ipPairs))
+}
+
+func SearchLocationByIp(ipString string) *LocationInfo {
+	ip := utils.StringIpToUint(ipString)
+	ipPairs := LocationDict.ipPairs
+	size := len(ipPairs)
+	if size == 0 || ip < ipPairs[0].BeginIp || ip > ipPairs[size - 1].EndIp {
+		return nil
+	}
+	left := 0
+	right := size - 1
+	for left <= right {
+		mid := (left + right) / 2
+		if ip >= ipPairs[mid].BeginIp && ip <= ipPairs[mid].EndIp {
+			return LocationDict.ipLocationMap[ipPairs[mid]];
+		} else if ip < ipPairs[mid].BeginIp {
+			right = mid - 1
+		} else if ip > ipPairs[mid].EndIp {
+			left = mid + 1
+		}
+	}
+	return nil
+}
+
+func loadGeoLocation(fileName string) map[uint64]*GeoLocationInfo {
+	dictFile, err := os.Open(fileName)
+	if err != nil {
+		fmt.Printf("open file error, name=%s\n", fileName)
+		return nil
+	}
+	defer dictFile.Close()
+
+	geoLocationMap := make(map[uint64]*GeoLocationInfo)
+	br := bufio.NewReader(dictFile)
+	lineNum := 0
+	for {
+		line, _, err := br.ReadLine()
+		if err == io.EOF {
+			break
+		}
+		lineString := string(line)
+		if len(lineString) == 0 || lineString[0] == '#' {
+			continue
+		}
+		lines := strings.Split(lineString, ",")
+		if len(lines) != 9 {
+			fmt.Printf("invalid format, file=%s, line=%s\n",
+				fileName, lineString)
+			continue
+		}
+		locId, _ := strconv.ParseUint(lines[0], 10, 64)
+		geoLocationInfo := &GeoLocationInfo{
+			LocId: locId,
+			Country: strings.Trim(lines[1], "\""),
+			City: strings.Trim(lines[3], "\""),
+		}
+		geoLocationMap[locId] = geoLocationInfo
+		lineNum++
+	}
+	fmt.Printf("load dict success, file=%s, lineNum=%d\n", fileName, lineNum)
+	return geoLocationMap
+}

src/main/main.go

@@ -7,7 +7,10 @@ import (
 )
 
 func main() {
-	adserver.ReadAdDict()
+	adserver.LoadLocationDict(
+		"./data/GeoLiteCity-Blocks.csv",
+		"./data/GeoLiteCity-Location.csv")
+	adserver.ReadAdDict("./data/ad_info.txt")
 	http.HandleFunc("/ad/search", adhandler.SearchHandler)
 	http.HandleFunc("/ad/impression",adhandler.ImpressionHandler)
 	http.HandleFunc("/ad/click",adhandler.ClickHandler)

src/utils/ip_pair.go

@@ -0,0 +1,42 @@
+package utils
+
+import (
+	"strings"
+	"strconv"
+)
+
+type IpPair struct {
+	BeginIp uint32
+	EndIp uint32
+}
+
+type IpPairs []IpPair
+
+func (p IpPairs) Len() int  {
+	return len(p)
+}
+
+func (p IpPairs) Less(i, j int) bool {
+	if p[i].BeginIp == p[j].BeginIp {
+		return p[i].EndIp < p[j].EndIp
+	} else {
+		return p[i].BeginIp < p[j].BeginIp
+	}
+}
+
+func (p IpPairs) Swap(i, j int)  {
+	p[i], p[j] = p[j], p[i]
+}
+
+func StringIpToUint(ipstring string) uint32 {
+	ipSegs := strings.Split(ipstring, ".")
+	var ipInt uint32 = 0
+	var pos uint32 = 24
+	for _, ipSeg := range ipSegs {
+		tempInt, _ := strconv.ParseUint(ipSeg, 10, 32)
+		intSeg := (uint32(tempInt)) << pos
+		ipInt = ipInt | intSeg
+		pos -= 8
+	}
+	return ipInt
+}

src/utils/utils.go

@@ -1,15 +1,18 @@
 package utils
+
 import(
   "os"
   "strconv"
   "time"
 )
+
 func CheckFileIsExist(filename string) (bool) {
 	if _, err := os.Stat(filename); os.IsNotExist(err) {
 		return false;
 	}
 	return true;
 }
+
 func GetLogFileName(filename,logpath string) string {
 	dateStr := strconv.Itoa(time.Now().Year()) + strconv.Itoa(int(time.Now().Month())) + strconv.Itoa(time.Now().Day())+strconv.Itoa(time.Now().Hour())
     var logFileName string