使用redis实现快速查询IP所属地

在做一些业务数据统计时,或多或少都会有这么一种需求,需要统计流量来自哪里,而在一般的应用系统中,和地区相关的数据就只有IP了,而上周笔者下载的IP库大概有300万记录,geolite2-开源数据库,如果使用传统的关系数据库是很难实现的,就算实现了查询速度也肯定是不能忍受的,这个时候redis就能非常好的解决这个问题。

思路

将IP地址转换成一个整数值,作为ShortedSet的分值与城市ID形成映射,然后将需要查询的IP通过相同的函数转换成一个整数值,查询出集合中分值比需要查询的IP小的集合中的最大值就是该IP所对应的结果。

程序清单

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
import com.google.gson.Gson;
import com.onway.common.model.dto.LocationDTO;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Stream;
/**
* Created by LiangT on 2017/7/10.
*/
public class IpLookup {
private static AtomicLong count = new AtomicLong(0);
private static JedisPool pool;
private static Gson gson = new Gson();
static {
JedisPoolConfig config = new JedisPoolConfig();
pool = new JedisPool(config, "118.178.135.106", 6479, 3000, "123456");
}
public static void main(String[] args) {
importResourceToRedis("/GeoLite2-City-Blocks-IPv4.csv", IpLookup::handleIpLine);
importResourceToRedis("/GeoLite2-City-Locations-zh-CN.csv", IpLookup::handleCityLine);
LocationDTO location = findIp2Location("183.5.15.218");
System.out.println(location);
}
private static void importResourceToRedis(String resource, Consumer<? super String[]> action) {
try (Stream<String> lines = Files
.lines(Paths.get(IpLookup.class.getResource(resource).toURI()))) {
lines.parallel().map(s -> s.split(",", -1)).forEach(action);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void handleIpLine(String[] line) {
String startIp = line.length > 1 ? line[0] : "";
if (startIp.toLowerCase().indexOf('i') != -1) {
return;
}
double score;
if (startIp.indexOf('.') != -1) {
String[] split = startIp.split("/", -1);
score = ipToScore(split[0]);
} else {
try {
score = Integer.parseInt(startIp, 10);
} catch (NumberFormatException nfe) {
return;
}
}
String cityId = line[1] + '_' + count.getAndIncrement();
Jedis jedis = null;
try {
jedis = pool.getResource();
jedis.zadd("ip2cityid", score, cityId);
} catch (Exception e) {
e.printStackTrace();
} finally {
jedis.close();
}
}
public static void handleCityLine(String[] line) {
if (line.length < 10 || !Character.isDigit(line[0].charAt(0))) {
return;
}
String cityId = line[0];
String json = gson.toJson(LocationDTO.builder().city(line[10])
.continent(line[3])
.province(line[7])
.country(line[5])
.build());
Jedis jedis = null;
try {
jedis = pool.getResource();
jedis.hset("cityid2city:", cityId, json);
} catch (Exception e) {
e.printStackTrace();
} finally {
jedis.close();
}
}
public static double ipToScore(String ipAddress) {
double score = 0;
for (String v : ipAddress.split("\\.")) {
score = score * 256 + Double.parseDouble(v);
}
return score;
}
public static LocationDTO findIp2Location(Jedis conn, String ipAddress) {
double score = ipToScore(ipAddress);
Set<String> results = conn.zrevrangeByScore("ip2cityid:", score, 0, 0, 1);
if (results.size() == 0) {
return null;
}
String cityId = results.iterator().next();
cityId = cityId.substring(0, cityId.indexOf('_'));
return gson.fromJson(conn.hget("cityid2city:", cityId), LocationDTO.class);
}
}