目前类似搜狗输入法、百度输入法等最大的亮点是有较好的词库,而这些词库除了用在搜索上还可以用在及其场景,比如说分词。目前每种输入法的词库都有自己的格式,今天我们就来研究下,如何反编译这些词库,将这些词库使用到其他的场景。
目录
搜狗输入法词库解析
搜狗输入法提供的词库下载下来是scel格式的,在使用前把他转成txt,以下为转换方法:
# -*- coding: utf-8 -*- import struct import sys # 搜狗的scel词库就是保存的文本的unicode编码,每两个字节一个字符(中文汉字或者英文字母) #找出其每部分的偏移位置即可 #主要两部分 #1.全局拼音表,貌似是所有的拼音组合,字典序 # 格式为(index,len,pinyin)的列表 # index: 两个字节的整数 代表这个拼音的索引 # len: 两个字节的整数 拼音的字节长度 # pinyin: 当前的拼音,每个字符两个字节,总长len # #2.汉语词组表 # 格式为(same,py_table_len,py_table,{word_len,word,ext_len,ext})的一个列表 # same: 两个字节 整数 同音词数量 # py_table_len: 两个字节 整数 # py_table: 整数列表,每个整数两个字节,每个整数代表一个拼音的索引 # # word_len:两个字节 整数 代表中文词组字节数长度 # word: 中文词组,每个中文汉字两个字节,总长度word_len # ext_len: 两个字节 整数 代表扩展信息的长度,好像都是10 # ext: 扩展信息 前两个字节是一个整数(不知道是不是词频) 后八个字节全是0 # # {word_len,word,ext_len,ext} 一共重复same次 同音词 相同拼音表 #拼音表偏移, startPy = 0x1540 #汉语词组表偏移 startChinese = 0x2628 #全局拼音表 GPy_Table = {} #解析结果 #元组(词频,拼音,中文词组)的列表 GTable = [] def byte2str(data): '''''将原始字节码转为字符串''' i = 0 length = len(data) ret = u'' while i < length: x = data[i] + data[i + 1] t = unichr(struct.unpack('H', x)[0]) if t == u'\r': ret += u'\n' elif t != u' ': ret += t i += 2 return ret #获取拼音表 def getPyTable(data): if data[0:4] != "\x9D\x01\x00\x00": return None data = data[4:] pos = 0 length = len(data) while pos < length: index = struct.unpack('H', data[pos] + data[pos + 1])[0] #print index, pos += 2 l = struct.unpack('H', data[pos] + data[pos + 1])[0] #print l, pos += 2 py = byte2str(data[pos:pos + l]) #print py GPy_Table[index] = py pos += l #获取一个词组的拼音 def getWordPy(data): pos = 0 length = len(data) ret = u'' while pos < length: index = struct.unpack('H', data[pos] + data[pos + 1])[0] ret += GPy_Table[index] pos += 2 return ret #获取一个词组 def getWord(data): pos = 0 length = len(data) ret = u'' while pos < length: index = struct.unpack('H', data[pos] + data[pos + 1])[0] ret += GPy_Table[index] pos += 2 return ret #读取中文表 def getChinese(data): #import pdb #pdb.set_trace() pos = 0 length = len(data) while pos < length: #同音词数量 same = struct.unpack('H', data[pos] + data[pos + 1])[0] #print '[same]:',same, #拼音索引表长度 pos += 2 py_table_len = struct.unpack('H', data[pos] + data[pos + 1])[0] #拼音索引表 pos += 2 py = getWordPy(data[pos: pos + py_table_len]) #中文词组 pos += py_table_len for i in xrange(same): #中文词组长度 c_len = struct.unpack('H', data[pos] + data[pos + 1])[0] #中文词组 pos += 2 word = byte2str(data[pos: pos + c_len]) #扩展数据长度 pos += c_len ext_len = struct.unpack('H', data[pos] + data[pos + 1])[0] #词频 pos += 2 count = struct.unpack('H', data[pos] + data[pos + 1])[0] #保存 GTable.append((count, py, word)) #到下个词的偏移位置 pos += ext_len def deal(file_name): print '-' * 60 f = open(file_name, 'rb') data = f.read() f.close() if data[0:12] != "\x40\x15\x00\x00\x44\x43\x53\x01\x01\x00\x00\x00": print "确认你选择的是搜狗(.scel)词库?" sys.exit(0) #pdb.set_trace() print "词库名:", byte2str(data[0x130:0x338]) #.encode('GB18030') print "词库类型:", byte2str(data[0x338:0x540]) #.encode('GB18030') print "描述信息:", byte2str(data[0x540:0xd40]) #.encode('GB18030') print "词库示例:", byte2str(data[0xd40:startPy]) #.encode('GB18030') getPyTable(data[startPy:startChinese]) getChinese(data[startChinese:]) if __name__ == '__main__': #将要转换的词库添加在这里就可以了 f = "搜狗标准词库.scel".decode("utf-8") deal(f) #保存结果 f = open('sougou.txt', 'w') for count, py, word in GTable: #GTable保存着结果,是一个列表,每个元素是一个元组(词频,拼音,中文词组),有需要的话可以保存成自己需要个格式 #我没排序,所以结果是按照上面输入文件的顺序 f.write(unicode('{%(count)s}' % {'count': count} + py + ' ' + word).encode('GB18030')) #最终保存文件的编码,可以自给改 f.write('\n') f.close()
其他参考资料:
百度输入法词库解析
#!/usr/bin/env python # -*- coding: utf-8 -*- import struct from tools import * class bdict(BaseDictFile): def __init__(self): BaseDictFile.__init__(self) # 文件头 self.head = 'biptbdsw' # 文件终点偏移 self.offset = 0x60 self.end_position = 0 # 词表偏移 self.dict_start = 0x350 self.shengmu = [ "c", "d", "b", "f", "g", "h", "ch", "j", "k", "l", "m", "n", "", "p", "q", "r", "s", "t", "sh", "zh", "w", "x", "y", "z"] self.yunmu = [ "uang", "iang", "iong", "ang", "eng", "ian", "iao", "ing", "ong", "uai", "uan", "ai", "an", "ao", "ei", "en", "er", "ua", "ie", "in", "iu", "ou", "ia", "ue", "ui", "un", "uo", "a", "e", "i", "o", "u", "v"] def _get_word_len(self, data, pos = 0): # 单词长度 length = struct.unpack('I', data[pos:pos+4])[0] char = struct.unpack('B', data[pos+4])[0] pure_english = False if 0x41 <= char <= 0x7a: pure_english = True #print(repr(data[pos:pos+length*3+1])) return (length, pure_english) def _get_word(self, data, length = 0, pure_english = False): pos = 0 word = Word() pinyin = [] for i in xrange(length): char = struct.unpack('B', data[pos])[0] pos += 1 if pure_english: # 如果读取到的首个hex值落在字母区域(0x41 ~ 0x7a)内,说明是纯英文,后面的内容直接读取即可 pinyin.append(chr(char).lower()) word.value += chr(char) else: if char == 0xff: # 声母部分如果是'\xff',说明这是中英文混输的英文字母,不需要做拼音词表转换,直接读取韵母部分即可 sm = '' ym = struct.unpack('c', data[pos])[0] pos += 1 pinyin.append('' + ym) else: sm = char ym = struct.unpack('B', data[pos])[0] pos += 1 pinyin.append(self.shengmu[sm] + self.yunmu[ym]) # try: # pinyin.append(self.shengmu[sm] + self.yunmu[ym]) # except IndexError: # print(repr(data)) # raise IndexError if pure_english: word.pinyin = ' '.join(pinyin) else: word.pinyin = ' '.join(pinyin) hanzi = byte2str(data[pos:pos+ length*2]) pos = pos+ length*2 word.value = hanzi.encode('utf-8') return word def get_dict_info(self, data): self.end_position = struct.unpack('I', data[self.offset:self.offset+4])[0] def load(self, filename): f = open(filename,'rb') data = f.read() f.close() if data[0:8] != self.head: print "It's not a bdict file" sys.exit(1) return self.read(data) def read(self, data): self.get_dict_info(data) pos = self.dict_start while pos < self.end_position: # print(pos, self.end_position) # if pos >= self.end_position: # break # 这里的长度是算字数的,实际长度是 拼音数(length * 2) + 字符长度(length * 2) length, pure_english = self._get_word_len(data, pos = pos) pos += 4 #print('*'*60) #print(repr(data[pos:pos+length*4])) if pure_english: word = self._get_word(data[pos:pos+length], length = length, pure_english = True) pos = pos + length else: word = self._get_word(data[pos:pos+length*4], length = length) pos = pos + length * 4 if word.value: if self.dictionary.has_key(word.pinyin): self.dictionary[word.pinyin].append(word) else: self.dictionary[word.pinyin] = [] self.dictionary[word.pinyin].append(word) return self.dictionary
其他参考资料:
QQ拼音输入法
JAVA版本:
/* Copyright (c) 2010 Xiaoyun Zhu * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ import java.io.ByteArrayOutputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.util.Arrays; import java.util.zip.InflaterOutputStream; /** * QQ Pinyin IME QPYD File Reader * * <pre> * QPYD Format overview: * * General Information: * - Chinese characters are all encoded with UTF-16LE. * - Pinyin are encoded in ascii (or UTF-8). * - Numbers are using little endian byte order. * * QPYD hex analysis: * - 0x00 QPYD file identifier * - 0x38 offset of compressed data (word-pinyin-dictionary) * - 0x44 total words in qpyd * - 0x60 start of header information * * Compressed data analysis: * - zip/standard (beginning with 0x789C) is used in (all analyzed) qpyd files * - data is divided in two parts * -- 1. offset and length information (16 bytes for each pinyin-word pair) * 0x06 offset points to first pinyin * 0x00 length of pinyin * 0x01 length of word * -- 2. actual data * Dictionary data has the form ((pinyin)(word))* with no separators. * Data can only be read using offset and length information. * * </pre> * */ public class QQPinyinQpydReader { public static void main(final String[] args) throws IOException { // download from http://dict.py.qq.com/list.php final String qqydFile = "D:\\test.qpyd"; // read qpyd into byte array final ByteArrayOutputStream dataOut = new ByteArrayOutputStream(); try (RandomAccessFile file = new RandomAccessFile(qqydFile, "r"); final FileChannel fChannel = file.getChannel();) { fChannel.transferTo(0, fChannel.size(), Channels.newChannel(dataOut)); } // qpyd as bytes final ByteBuffer dataRawBytes = ByteBuffer.wrap(dataOut.toByteArray()); dataRawBytes.order(ByteOrder.LITTLE_ENDIAN); System.out.println("文件: " + qqydFile); // read info of compressed data final int startZippedDictAddr = dataRawBytes.getInt(0x38); final int zippedDictLength = dataRawBytes.limit() - startZippedDictAddr; // qpys as UTF-16LE string final String dataString = new String(Arrays.copyOfRange(dataRawBytes.array(), 0x60, startZippedDictAddr), "UTF-16LE"); // print header System.out.println("名称:" + QQPinyinQpydReader.substringBetween(dataString, "Name: ", "\r\n")); System.out.println("类型:" + QQPinyinQpydReader.substringBetween(dataString, "Type: ", "\r\n")); System.out.println("子类型:" + QQPinyinQpydReader.substringBetween(dataString, "FirstType: ", "\r\n")); System.out.println("词库说明:" + QQPinyinQpydReader.substringBetween(dataString, "Intro: ", "\r\n")); System.out.println("词库样例:" + QQPinyinQpydReader.substringBetween(dataString, "Example: ", "\r\n")); System.out.println("词条数:" + dataRawBytes.getInt(0x44)); // read zipped qqyd dictionary into byte array dataOut.reset(); try (InflaterOutputStream inflater = new InflaterOutputStream(dataOut);) { Channels.newChannel(inflater).write(ByteBuffer.wrap(dataRawBytes.array(), startZippedDictAddr, zippedDictLength)); } // uncompressed qqyd dictionary as bytes final ByteBuffer dataUnzippedBytes = ByteBuffer.wrap(dataOut.toByteArray()); dataUnzippedBytes.order(ByteOrder.LITTLE_ENDIAN); // for debugging: save unzipped data to *.unzipped file try (FileOutputStream out = new FileOutputStream(qqydFile + ".unzipped");) { Channels.newChannel(out).write(dataUnzippedBytes); System.out.println("压缩数据:0x" + Integer.toHexString(startZippedDictAddr) + " (解压前:" + zippedDictLength + " B, 解压后:" + dataUnzippedBytes.limit() + " B)"); } // stores the start address of actual dictionary data int unzippedDictStartAddr = -1; final byte[] byteArray = dataUnzippedBytes.array(); dataUnzippedBytes.position(0); while ((unzippedDictStartAddr == -1) || (dataUnzippedBytes.position() < unzippedDictStartAddr)) { // read word final int pinyinLength = dataUnzippedBytes.get() & 0xff; final int wordLength = dataUnzippedBytes.get() & 0xff; dataUnzippedBytes.getInt(); // garbage final int pinyinStartAddr = dataUnzippedBytes.getInt(); final int wordStartAddr = pinyinStartAddr + pinyinLength; if (unzippedDictStartAddr == -1) { unzippedDictStartAddr = pinyinStartAddr; System.out.println("词库地址(解压后):0x" + Integer.toHexString(unzippedDictStartAddr) + "\n"); } final String pinyin = new String(Arrays.copyOfRange(byteArray, pinyinStartAddr, pinyinStartAddr + pinyinLength), "UTF-8"); final String word = new String(Arrays.copyOfRange(byteArray, wordStartAddr, wordStartAddr + wordLength), "UTF-16LE"); System.out.println(word + "\t" + pinyin); } } public static final String substringBetween(final String text, final String start, final String end) { final int nStart = text.indexOf(start); final int nEnd = text.indexOf(end, nStart + 1); if ((nStart != -1) && (nEnd != -1)) { return text.substring(nStart + start.length(), nEnd); } else { return null; } } }
其他参考资料: