正向最大匹配

# -*- coding:utf-8 -*-

CODEC='utf-8'

def u(s, encoding):

'converted other encoding to unicode encoding'

if isinstance(s, unicode):

return s

else:

return unicode(s, encoding)

def fwd_mm_seg(wordDict, maxLen, str):

'forward max match segment'

wordList = []

segStr = str

segStrLen = len(segStr)

for word in wordDict:

print 'word: ', word

print "\n"

while segStrLen > 0:

if segStrLen > maxLen:

wordLen = maxLen

else:

wordLen = segStrLen

subStr = segStr[0:wordLen]

print "subStr: ", subStr

while wordLen > 1:

if subStr in wordDict:

print "subStr1: %r" % subStr

break

else:

print "subStr2: %r" % subStr

wordLen = wordLen - 1

subStr = subStr[0:wordLen]

# print "subStr3: ", subStr

wordList.append(subStr)

segStr = segStr[wordLen:]

segStrLen = segStrLen - wordLen

for wordstr in wordList:

print "wordstr: ", wordstr

return wordList

def main():

fp_dict = open('words.dic')

wordDict = {}

for eachWord in fp_dict:

wordDict[u(eachWord.strip(), 'utf-8')] = 1

segStr = u'你好世界hello world'

print segStr

wordList = fwd_mm_seg(wordDict, 10, segStr)

print "==".join(wordList)

if __name__ == '__main__':

main()

逆向最大匹配

# -*- coding:utf-8 -*-

def u(s, encoding):

'converted other encoding to unicode encoding'

if isinstance(s, unicode):

return s

else:

return unicode(s, encoding)

CODEC='utf-8'

def bwd_mm_seg(wordDict, maxLen, str):

'forward max match segment'

wordList = []

segStr = str

segStrLen = len(segStr)

for word in wordDict:

print 'word: ', word

print "\n"

while segStrLen > 0:

if segStrLen > maxLen:

wordLen = maxLen

else:

wordLen = segStrLen

subStr = segStr[-wordLen:None]

print "subStr: ", subStr

while wordLen > 1:

if subStr in wordDict:

print "subStr1: %r" % subStr

break

else:

print "subStr2: %r" % subStr

wordLen = wordLen - 1

subStr = subStr[-wordLen:None]

# print "subStr3: ", subStr

wordList.append(subStr)

segStr = segStr[0: -wordLen]

segStrLen = segStrLen - wordLen

wordList.reverse()

for wordstr in wordList:

print "wordstr: ", wordstr

return wordList

def main():

fp_dict = open('words.dic')

wordDict = {}

for eachWord in fp_dict:

wordDict[u(eachWord.strip(), 'utf-8')] = 1

segStr = ur'你好世界hello world'

print segStr

wordList = bwd_mm_seg(wordDict, 10, segStr)

print "==".join(wordList)

if __name__ == '__main__':

main()

以上這篇python正向最大匹配分詞和逆向最大匹配分詞的實例就是小編分享給大家的全部內(nèi)容了，希望能給大家一個參考，也希望大家多多支持腳本之家。

總結(jié)

以上是生活随笔為你收集整理的python最大分词_python正向最大匹配分词和逆向最大匹配分词的实例的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。