三人行,必有我师焉。择其善者而从之,其不善者而改之。——孔子
/**
   * mapPartitions(func)
   用法: rdd2=rdd1.mapPartitions(func)
   语义: 对rdd1中每个分区进行func操作,输出新的一组数据
   * */
  public static void mapPartitions() {

      //创建SparkConf
      SparkConf conf = new SparkConf().setAppName("map").setMaster("local");
      //创建javaSparkContext
      JavaSparkContext sc = new JavaSparkContext(conf);
      JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4));
      JavaRDD<Tuple2<Integer, Integer>> tuple2JavaRDD = rdd.mapPartitions(new FlatMapFunction<Iterator<Integer>, Tuple2<Integer, Integer>>() {
          @Override
          public Iterator<Tuple2<Integer, Integer>> call(Iterator<Integer> it) throws Exception {
              ArrayList<Tuple2<Integer, Integer>> tuple2s = new ArrayList<>();
              while (it.hasNext()) {
                  Integer newValue=0;
                  Integer value=it.next();
                  if(value%2==0)
                      newValue=value*value;
                  else
                      newValue=value;
                  tuple2s.add(new Tuple2<>(value,newValue));

              }
              return tuple2s.iterator();
          }
      });
      tuple2JavaRDD.foreach(new VoidFunction<Tuple2<Integer, Integer>>() {
          @Override
          public void call(Tuple2<Integer, Integer> integerIntegerTuple2) throws Exception {
              System.out.println(integerIntegerTuple2);
          }
      });
  }
答案: B
;
版权声明: 本文为智客工坊「楠木大叔」的原创文章,遵循CC 4.0 BY-SA版权协议,转载请附上原文出处链接及本声明。

results matching ""

    No results matching ""