大数据集:频繁测试成员关系

   1 ~/algorithm/censor $ cat lib/multi_pattern_search/keyword2group.py
   2 #coding:utf-8
   3 from array import array
   4 from collections import defaultdict
   5 from multi_pattern_search import MultiPatternSearch
   6 
   7 
   8 class CensorKeyword(object):
   9     def __init__(self):
  10         pass
  11 
  12     def _reload(self):
  13         self.group_name_list = []
  14         self.single_keyword_to_cat_id = {}
  15         self.group_keyword = []
  16         self.group_keyword_to_group_id = {}
  17         self.group_id_to_cat_id = array('I')
  18         self.group_keyword_length = array('B')
  19         self.single_search = MultiPatternSearch()
  20         self.group_search = MultiPatternSearch()
  21 
  22     def _init_search(self):
  23         for i in self.single_keyword_to_cat_id.keys():
  24             self.single_search.add_keyword(i)
  25         group_keyword_set = set()
  26         for i in self.group_keyword:
  27             for k in i:
  28                 group_keyword_set.add(k)
  29         for i in group_keyword_set:
  30             self.group_search.add_keyword(i)
  31 
  32     def _keyword_to_group_id(self):
  33         #词做倒排
  34         group_keyword_to_group_id = self.group_keyword_to_group_id
  35         for pos, group_keyword_list in enumerate(self.group_keyword):
  36             for keyword in group_keyword_list:
  37                 if keyword not in group_keyword_to_group_id:
  38                     group_keyword_to_group_id[keyword] = array('I')
  39                 group_keyword_to_group_id[keyword].append(pos)
  40 
  41     def load_txt(self, txt):
  42         self._reload()
  43         SINGE_KEYWORD = 0
  44         GROUP_KEYWORD = 1
  45         txt = txt.split("\n")
  46         state = SINGE_KEYWORD
  47         group_keyword = set()
  48         #处理文本
  49         for line in txt:
  50             line = line.strip()
  51             if not line:
  52                 continue
  53             if line.startswith(">>>"):
  54                 cat_id = len(self.group_name_list)
  55                 group_name_list = line[3:].strip()
  56                 self.group_name_list.append(group_name_list)
  57                 state = SINGE_KEYWORD
  58             elif len(line) == 1:
  59                 if line == '{':
  60                     state = GROUP_KEYWORD
  61                 elif line == "}":
  62                     state = SINGE_KEYWORD
  63                     if group_keyword:
  64                         group_keyword = tuple(group_keyword)
  65                         self.group_keyword.append(group_keyword)
  66                         self.group_id_to_cat_id.append(cat_id)
  67                         self.group_keyword_length.append(len(group_keyword))
  68                         group_keyword = set()
  69             elif state == SINGE_KEYWORD:
  70                 self.single_keyword_to_cat_id[line] = cat_id
  71             elif state == GROUP_KEYWORD:
  72                 group_keyword.add(line)
  73 
  74         self._keyword_to_group_id()
  75         self._init_search()
  76 
  77     def which_group_id(self, text):
  78         total_group = len(self.group_name_list)
  79         group = total_group
  80 
  81         single_search = self.single_search.count(text)
  82         all_keys =        single_search.keys()
  83         if single_search:
  84             for key in single_search.keys():
  85                 cat_id = self.single_keyword_to_cat_id[key]
  86                 if cat_id < group:
  87                     group = cat_id
  88 
  89         group_search = self.group_search.count(text)
  90         all_keys.extend(group_search.keys())
  91         group_count = defaultdict(int)
  92         for k in group_search.keys():
  93             group_id_list = self.group_keyword_to_group_id[k]
  94             for i in group_id_list:
  95                 group_count[i]+=1
  96 
  97         for k, v in group_count.iteritems():
  98             if self.group_keyword_length[k] <= v:
  99                 #返回最小的那个 事情比事后优先级高
 100                 cat_id = self.group_id_to_cat_id[k]
 101                 if cat_id < group:
 102                     group = cat_id
 103         if group < total_group:
 104             return group, all_keys
 105 
 106     def which_group_name(self, text):
 107         pos = self.which_group_id(text)
 108         if pos is not None:
 109             pos , keywords = pos
 110             keywords = " ".join(keywords)
 111             return self.group_name_list[pos], keywords
 112 
 113 
 114 if __name__ == "__main__":
 115     test_input = """
 116 >>> 广告事前
 117     贵公司
 118 {
 119     办证
 120     电话
 121 }
 122 {
 123     是
 124     网站
 125     促销
 126 }
 127 
 128 
 129 >>> 敏感文章事前
 130 
 131 
 132 >>> 广告事后
 133     开发票
 134 {
 135     网站
 136     代购
 137 }
 138     的
 139 
 140 
 141 >>> 敏感文章事后
 142 
 143 """
 144     censor_keyword = CensorKeyword()
 145     censor_keyword.load_txt(test_input)
 146     censor_keyword.load_txt(test_input)
 147     censor_keyword.load_txt(test_input)
 148     test = """
 149 牛奶@咖啡3月21日专场演出（免票）
 150 牛奶@咖啡3月21日专场演出（免票）
 151 活动介绍
 152 牛奶@咖啡自从巡演回来后一直没有在北京作专场演出，如果你对最近每次的小型演出还不过瘾，就到我们的这次专场上听个够吧！“一起不孤单！” 
 153 
 154 让更多的人来一起倾听牛奶@咖啡的歌声吧！如果你还不知道他们，去下面的网址了解一下吧！http://www.douban.com/group/milkcoffee/ 
 155 
 156 牛奶@咖啡3月21日周六下午1点半SOHO尚都专场，免票无需报名 
 157 
 158 地点：SOHO尚都西塔一层钢琴舞台（地铁一号线永安里站B口出倒126路公交芳草地站下车即是） 
 159 咨询电话：老徐：13801108173 
 160 活动网址：http://www.douban.com/event/10533960/
 161 同时还有市集可以逛：http://www.douban.com/event/10503500/
 162     """
 163 
 164     #print " ".join(censor_keyword.which_group_name("贵公司"))
 165     #print " ".join(censor_keyword.which_group_name("代购网站的"))
 166     print " ".join(censor_keyword.which_group_name(test))