场景
网站数据太多，速度太慢？某处逻辑反复的调用一个函数。这样的开销是没有必要的。如果我们能将函数的结果保存到内存中，或者某个能快速反应的DB中，那么我们网站的响应速度会得到大大的提高。适用情景：
- 某函数参数常不变，但是里面有大量的逻辑需要处理，如果每次运行都需要运行的话，那么将花费大量的时间，那么我们可以采取缓存的方式，将函数的返回值进行缓存，下来执行到此处的时候直接返回即可。
- 函数的参数是变化的，但并不是每次都在变。这时我们也可以采取此方式进行优化。
代码
缓存源码示例：1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165# encoding=utf8
import hashlib
import os
import time
from common.log_utils import getLogger
log = getLogger('mem_cache.py')
__all__ = ['get', 'set', 'memorize', 'memcache_client']
import memcache
import settings
memcache_client = memcache.Client(
['%s:%s' % (settings.MEMCACHE_SERVER, str(settings.MEMCACHE_PORT))])
OBJ_DICT = {}
OBJ_DICT2 = {}
OBJ_EXPIRE_DICT = {}
OBJ_EXPIRE_DICT2 = {}
DEFAULT_EXPIRE_SECONDS = 300
DEFAULT_VERSION_EXPIRE_SECONDS = 60 * 60 * 24
MEMORY_CLEANER_PERIOD = 600
OBJ_VERSION_DICT = {}
def set(key, value, time=settings.MEMCACHE_EXPIRE_TIME):
"""time为过期时间，以秒为单位"""
return memcache_client.set(str(key), value, time)
def get(key):
return memcache_client.get(str(key))
def set_remote_obj_version(key, version):
return memcache_client.set(str(key), version, DEFAULT_VERSION_EXPIRE_SECONDS)
def get_remote_obj_version(key):
return memcache_client.get(str(key)) or 0
def set_local_obj_version(key, version):
OBJ_VERSION_DICT[key] = version
def get_local_obj_version(key):
return OBJ_VERSION_DICT.get(key) or 0
def version_expired(key):
local_version = get_local_obj_version(key)
remote_version = get_remote_obj_version(key)
return local_version != remote_version
def delete_all():
delete_list = []
for key, value in OBJ_DICT.iteritems():
delete_list.append(key)
memcache_client.delete_multi(delete_list)
def start_memory_cleaner(count_down=MEMORY_CLEANER_PERIOD):
while True:
time.sleep(count_down)
keys_to_remove = []
for key, value in OBJ_EXPIRE_DICT.iteritems():
if OBJ_EXPIRE_DICT[key] < int(round(time.time())):
keys_to_remove.append(key)
for key in keys_to_remove:
try:
del OBJ_DICT[key]
del OBJ_EXPIRE_DICT[key]
del OBJ_VERSION_DICT[key]
except KeyError:
pass
for key, value in OBJ_EXPIRE_DICT2.iteritems():
if OBJ_EXPIRE_DICT2[key] < int(round(time.time())):
keys_to_remove.append(key)
for key in keys_to_remove:
try:
del OBJ_DICT[key]
del OBJ_EXPIRE_DICT2[key]
del OBJ_VERSION_DICT[key]
except KeyError:
pass
log.debug('%s keys deleted, %s keys remain' %
(len(keys_to_remove), len(OBJ_DICT)),)
memory_cleaner_thread_started = False
def start_memory_cleaner_thread():
global memory_cleaner_thread_started
import threading
if not memory_cleaner_thread_started:
threading.Thread(target=start_memory_cleaner, args=(),
name='memory_cleaner_thread').start()
memory_cleaner_thread_started = True
def memorize(function):
"""内存缓存, 用于不定参数的函数
Usage:
def xxx(key1,key2,refresh=False)
return instance
def xxx(key1,key2,key2,x1=xx,x2=yyy)
return instance
"""
def helper(*args, **kwargs):
refresh = False
expire = None
if 'refresh' in kwargs:
refresh = kwargs['refresh']
if 'expire' in kwargs:
expire = kwargs['expire']
# make cache key
args_str = []
for arg in args:
if isinstance(arg, unicode):
args_str.append(arg.encode('utf-8'))
else:
args_str.append(str(arg))
key = function.__name__ + hashlib.md5('#'.join(args_str)).hexdigest()
# end make cache key
remote_version = get_remote_obj_version(key)
ret_obj = None
if refresh:
# 1.
remote_version += 1
set_remote_obj_version(key, remote_version)
# #当主动刷新时，直接调用；且主动更新本地缓存（可以不做，只是其他程序调用时进入#3段，重复调用一次再更新本地缓存）
# ret_obj = function(*args, **kwargs)
# OBJ_DICT[key] = ret_obj
# set_local_obj_version(key, remote_version)
else:
if remote_version == 0:
# 2
ret_obj = function(*args, **kwargs)
OBJ_DICT[key] = ret_obj
set_remote_obj_version(key, 1)
set_local_obj_version(key, 1)
else:
local_version = get_local_obj_version(key)
if local_version != remote_version:
# 3
ret_obj = function(*args, **kwargs)
OBJ_DICT[key] = ret_obj
set_local_obj_version(key, remote_version)
else:
# 4
if key in OBJ_DICT:
# 5
ret_obj = OBJ_DICT[key]
else:
ret_obj = function(*args, **kwargs)
OBJ_DICT[key] = ret_obj
return ret_obj
return helper
@memorize
def test(x):
x = os.path.join("D:/dev/project/BI/saic/csvdata", x)
return file(x).read()
@memorize
def test2(x, ab, refresh=False):
x = os.path.join("D:/dev/project/BI/saic/csvdata", x)
return file(x).read()
if __name__ == '__main__':
x = 'utils.py'
txt = test2(x, 'ss', refresh=True)
txt = test2(x, 'ss')
使用
以Python装饰起为例，使用时直接在函数上加上装饰器即可使用。1
2
3
4@memorize
def get_current_df(raw_data_excel, key):
current_df = pd.read_hdf(raw_data_excel, key=key)
return current_df