大数据架构——词频分析 | 文本搜索(Spark 版)

from pyspark import SparkContext


def main():
    sc = SparkContext(appName='SparkWordCount')

    input_file = sc.textFile('hdfs:///Input.txt')
    counts = input_file.flatMap(lambda line: line.split()).map(lambda word: (word, 1)).reduceByKey(lambda a, b: a + b)
    counts.saveAsTextFile('output3')
    sc.stop()


if __name__ == '__main__':
    main()

运行：$spark-submit --master local word_count.py

找出含有指定字符串的电影。 数据集：movies
数据浏览：

from pyspark import SparkContext
import re
import sys


def main():
    if (len(sys.argv)) != 2:
        sys.stderr.write('Usage: {} '.format(sys.argv[0]))
        sys.exit()
    sc = SparkContext(appName='SparkWordCount')

    requested_movie = sc.broadcast(sys.argv[1])

    source_file = sc.textFile('/movies')

    titles = source_file.map(lambda line: line.split('|')[1])

    normalized_title = titles.map(lambda title: (re.sub(r's*(d{4})', '', title).lower(), title))

    matches = normalized_title.filter(lambda x: requested_movie.values in x[0])

    matching_titles = matches.map(lambda x: x[1]).distinct().collect()

    print('{} Matching titles found:'.format(len(matching_titles)))
    for title in matching_titles:
        print(title)

    sc.stop()


if __name__ == '__main__':
    main()

运行：$spark-submit text_search.py gold