NLineInputFormat的示例分析
小编给大家分享一下NLineInputFormat的示例分析,希望大家阅读完这篇文章之后都有所收获,下面让我们一起去探讨吧!
成都创新互联是专业的伊通网站建设公司,伊通接单;提供网站设计制作、成都网站制作,网页设计,网站设计,建网站,PHP网站建设等专业做网站服务;采用PHP框架,可快速的进行伊通网站开发网页制作和功能扩展;专业做搜索引擎喜爱的网站,专业的做网站团队,希望更多企业前来合作!
package com.test; import java.io.IOException; import java.util.Iterator; import java.util.StringTokenizer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; /** * TextInputFormat处理的数据来自于一个InputSplit。InputSplit根据块大小划分。 * 由于每条记录有长有短,所以,每个map任务处理的记录数都不一样 * NLineInputFormat决定每个map处理记录数是相同的 */ public class WordCountNL extends Configured implements Tool { public static class Map extends Mapper{ public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String line = value.toString(); StringTokenizer st = new StringTokenizer(line); while(st.hasMoreElements()) { context.write(new Text(st.nextElement().toString()), new IntWritable(1)); } } } public static class Combiner extends Reducer { public void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { int count = 0; Iterator it = values.iterator(); while(it.hasNext()) { count = count + it.next().get(); } context.write(key, new IntWritable(count)); } } public static class Reduce extends Reducer { public void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException { int count = 0; Iterator it = values.iterator(); while(it.hasNext()) { count = count + it.next().get(); } context.write(key, new IntWritable(count)); } } public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = this.getConf(); //设置每个map可以处理多少行数据 //conf.set("mapreduce.input.lineinputformat.linespermap", "1"); conf.set(NLineInputFormat.LINES_PER_MAP, "1"); Job job = new Job(conf); job.setJobName(WordCountNL.class.getSimpleName()); job.setJarByClass(WordCountNL.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.setMapperClass(Map.class); job.setCombinerClass(Combiner.class); job.setReducerClass(Reduce.class); job.setInputFormatClass(NLineInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.waitForCompletion(true); return job.isSuccessful()?0:1; } public static void main(String[] args) throws Exception { int exit = ToolRunner.run(new WordCount(), args); System.exit(exit); } }
看完了这篇文章,相信你对“NLineInputFormat的示例分析”有了一定的了解,如果想了解更多相关知识,欢迎关注创新互联行业资讯频道,感谢各位的阅读!
本文题目:NLineInputFormat的示例分析
文章位置:http://myzitong.com/article/jpgsjp.html