牛骨文教育服务平台(让学习变的简单)
博文笔记

hadoop中java.lang.OutOfMemoryError: Java heap space

创建时间:2014-04-08 投稿人: 浏览次数:198
1.这是dataguru第五周视频的代码
import java.io.IOException;
import java.io.InterruptedIOException;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;


import java.lang.ArrayIndexOutOfBoundsException;


public class Test_2 extends Configured implements Tool {
	enum Counter {
		LINESKIP, // error line
	}


	public static class Map extends
			Mapper<LongWritable, Text, NullWritable, Text> {
		public void map(LongWritable key, Text value, Context context)
				throws IOException, InterruptedException {
			String line = value.toString();
			try {


				String[] lineSplit = line.split(" ");


				String month = lineSplit[0];
				String time = lineSplit[1];
				String mac = lineSplit[6];
				Text out = new Text(month + "" + time + "" + mac);


				context.write(NullWritable.get(), out);


			} catch (ArrayIndexOutOfBoundsException e) {
				context.getCounter(Counter.LINESKIP).increment(1);
				return;
			}
		}


	}// end map


	public int run(String[] args) throws Exception {
		Configuration conf = getConf();
		Job job = new Job(conf, "Test_2");
		job.setJarByClass(Test_2.class);


		FileInputFormat.addInputPath(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));


		job.setMapperClass(Map.class);
		job.setOutputFormatClass(TextOutputFormat.class);
		job.setOutputKeyClass(NullWritable.class);
		job.setOutputValueClass(Text.class);
		job.waitForCompletion(true);
		return job.isSuccessful() ? 0 : 1;


	}// end run


	public static void main(String[] args) throws Exception {
		int res = ToolRunner.run(new Configuration(), new Test_2(), args);
		System.exit(res);


	}// end main


}// end Test_2 class
注意设置
如果出现安全模式错误Name node is in safe mode,则用hadoop dfsadmin -safemode leave
声明:该文观点仅代表作者本人,牛骨文系教育信息发布平台,牛骨文仅提供信息存储空间服务。