您可以传递一个callable作为
analyzer参数来完全控制令牌化，例如
>>> from pprint import pprint>>> import re>>> x = ['this is a foo bar', 'you are a foo bar black sheep']>>> def words_and_char_bigrams(text):... words = re.findall(r'w{3,}', text)... for w in words:... yield w... for i in range(len(w) - 2):... yield w[i:i+2]... >>> v = CountVectorizer(analyzer=words_and_char_bigrams)>>> pprint(v.fit(x).vocabulary_){'ac': 0, 'ar': 1, 'are': 2, 'ba': 3, 'bar': 4, 'bl': 5, 'black': 6, 'ee': 7, 'fo': 8, 'foo': 9, 'he': 10, 'hi': 11, 'la': 12, 'sh': 13, 'sheep': 14, 'th': 15, 'this': 16, 'yo': 17, 'you': 18}
