mahout0.8入门
mahout-distribution-0.8
命令行对应哪个类可以查看源码配置文件
创新互联公司是一家专业的成都网站建设公司,我们专注成都做网站、成都网站设计、网络营销、企业网站建设,友情链接,1元广告为企业客户提供一站式建站解决方案,能带给客户新的互联网理念。从网站结构的规划UI设计到用户体验提高,创新互联力求做到尽善尽美。
driver.classes.default.props
mahout的API
https://builds.apache.org/job/Mahout-Quality/javadoc/
mahout实战参考博客:
http://itindex.net/detail/45259-mahout-%E7%94%B5%E5%BD%B1-%E6%8E%A8%E8%8D%90%E7%B3%BB%E7%BB%9F
聚类算法
kmeans:无法消除离群点的影响
canopy:两个阈值t1和t2,且t1>t2,简单快速不太准确,可以消除离群点的影响,一般用来决定聚类中心数目k
Canopy聚类算法
http://my.oschina.net/liangtee/blog/125407
mahout canopy算法实战
http://blog.csdn.net/xyilu/article/details/9631677
分类Bayes(训练集,基于概率的)、文本分类算法(监督学习)
朴素贝叶斯分类器两种模型:
多项式模型,以单词打标签,粒度不一样
伯努利模型,以文档打标签
用于新闻分类:体育、娱乐
mahout中提供了一种将指定文件下的文件转换成sequenceFile的方式。
mahout seqdirectory --input /hive/hadoopuser/ --output /mahout/seq/ --charset UTF-8
二进制文件转换为向量
mahout seq2sparse
完成朴素贝叶斯分类(中文分词)
f.dataguru.cn/thread-244375-1-1.html
http://www.cnblogs.com/panweishadow/p/4320720.html
低版本中还是老的贝叶斯testclassifier
0.11已经是新贝叶斯
#Classification
#new bayes
org.apache.mahout.classifier.naivebayes.training.TrainNaiveBayesJob = trainnb : Train the Vector-based Bayes classifier
org.apache.mahout.classifier.naivebayes.test.TestNaiveBayesDriver = testnb : Test the Vector-based Bayes classifier
cbayes=ComplementaryNaiveBayes
TestNaiveBayesDriver源码
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.mahout.classifier.naivebayes.test; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.regex.Pattern; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; import org.apache.hadoop.util.ToolRunner; import org.apache.mahout.classifier.ClassifierResult; import org.apache.mahout.classifier.ResultAnalyzer; import org.apache.mahout.classifier.naivebayes.AbstractNaiveBayesClassifier; import org.apache.mahout.classifier.naivebayes.BayesUtils; import org.apache.mahout.classifier.naivebayes.ComplementaryNaiveBayesClassifier; import org.apache.mahout.classifier.naivebayes.NaiveBayesModel; import org.apache.mahout.classifier.naivebayes.StandardNaiveBayesClassifier; import org.apache.mahout.common.AbstractJob; import org.apache.mahout.common.HadoopUtil; import org.apache.mahout.common.Pair; import org.apache.mahout.common.commandline.DefaultOptionCreator; import org.apache.mahout.common.iterator.sequencefile.PathFilters; import org.apache.mahout.common.iterator.sequencefile.PathType; import org.apache.mahout.common.iterator.sequencefile.SequenceFileDirIterable; import org.apache.mahout.math.Vector; import org.apache.mahout.math.VectorWritable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Test the (Complementary) Naive Bayes model that was built during training * by running the iterating the test set and comparing it to the model */ public class TestNaiveBayesDriver extends AbstractJob { private static final Logger log = LoggerFactory.getLogger(TestNaiveBayesDriver.class); public static final String COMPLEMENTARY = "class"; //b for bayes, c for complementary private static final Pattern SLASH = Pattern.compile("/"); public static void main(String[] args) throws Exception { ToolRunner.run(new Configuration(), new TestNaiveBayesDriver(), args); } @Override public int run(String[] args) throws Exception { addInputOption(); addOutputOption(); addOption(addOption(DefaultOptionCreator.overwriteOption().create())); addOption("model", "m", "The path to the model built during training", true); addOption(buildOption("testComplementary", "c", "test complementary?", false, false, String.valueOf(false))); addOption(buildOption("runSequential", "seq", "run sequential?", false, false, String.valueOf(false))); addOption("labelIndex", "l", "The path to the location of the label index", true); Map> parsedArgs = parseArguments(args); if (parsedArgs == null) { return -1; } if (hasOption(DefaultOptionCreator.OVERWRITE_OPTION)) { HadoopUtil.delete(getConf(), getOutputPath()); } boolean sequential = hasOption("runSequential"); boolean succeeded; if (sequential) { runSequential(); } else { succeeded = runMapReduce(); if (!succeeded) { return -1; } } //load the labels Map labelMap = BayesUtils.readLabelIndex(getConf(), new Path(getOption("labelIndex"))); //loop over the results and create the confusion matrix SequenceFileDirIterable dirIterable = new SequenceFileDirIterable<>(getOutputPath(), PathType.LIST, PathFilters.partFilter(), getConf()); ResultAnalyzer analyzer = new ResultAnalyzer(labelMap.values(), "DEFAULT"); analyzeResults(labelMap, dirIterable, analyzer); log.info("{} Results: {}", hasOption("testComplementary") ? "Complementary" : "Standard NB", analyzer); return 0; } private void runSequential() throws IOException { boolean complementary = hasOption("testComplementary"); FileSystem fs = FileSystem.get(getConf()); NaiveBayesModel model = NaiveBayesModel.materialize(new Path(getOption("model")), getConf()); // Ensure that if we are testing in complementary mode, the model has been // trained complementary. a complementarty model will work for standard classification // a standard model will not work for complementary classification if (complementary){ Preconditions.checkArgument((model.isComplemtary()), "Complementary mode in model is different from test mode"); } AbstractNaiveBayesClassifier classifier; if (complementary) { classifier = new ComplementaryNaiveBayesClassifier(model); } else { classifier = new StandardNaiveBayesClassifier(model); } try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, getConf(), new Path(getOutputPath(), "part-r-00000"), Text.class, VectorWritable.class)) { SequenceFileDirIterable dirIterable = new SequenceFileDirIterable<>(getInputPath(), PathType.LIST, PathFilters.partFilter(), getConf()); // loop through the part-r-* files in getInputPath() and get classification scores for all entries for (Pair pair : dirIterable) { writer.append(new Text(SLASH.split(pair.getFirst().toString())[1]), new VectorWritable(classifier.classifyFull(pair.getSecond().get()))); } } } private boolean runMapReduce() throws IOException, InterruptedException, ClassNotFoundException { Path model = new Path(getOption("model")); HadoopUtil.cacheFiles(model, getConf()); //the output key is the expected value, the output value are the scores for all the labels Job testJob = prepareJob(getInputPath(), getOutputPath(), SequenceFileInputFormat.class, BayesTestMapper.class, Text.class, VectorWritable.class, SequenceFileOutputFormat.class); //testJob.getConfiguration().set(LABEL_KEY, getOption("--labels")); boolean complementary = hasOption("testComplementary"); testJob.getConfiguration().set(COMPLEMENTARY, String.valueOf(complementary)); return testJob.waitForCompletion(true); } private static void analyzeResults(Map labelMap, SequenceFileDirIterable dirIterable, ResultAnalyzer analyzer) { for (Pair pair : dirIterable) { int bestIdx = Integer.MIN_VALUE; double bestScore = Long.MIN_VALUE; for (Vector.Element element : pair.getSecond().get().all()) { if (element.get() > bestScore) { bestScore = element.get(); bestIdx = element.index(); } } if (bestIdx != Integer.MIN_VALUE) { ClassifierResult classifierResult = new ClassifierResult(labelMap.get(bestIdx), bestScore); analyzer.addInstance(pair.getFirst().toString(), classifierResult); } } } }
BayesTestMapper源码
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.mahout.classifier.naivebayes.test; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; import org.apache.mahout.classifier.naivebayes.AbstractNaiveBayesClassifier; import org.apache.mahout.classifier.naivebayes.ComplementaryNaiveBayesClassifier; import org.apache.mahout.classifier.naivebayes.NaiveBayesModel; import org.apache.mahout.classifier.naivebayes.StandardNaiveBayesClassifier; import org.apache.mahout.common.HadoopUtil; import org.apache.mahout.math.Vector; import org.apache.mahout.math.VectorWritable; import java.io.IOException; import java.util.regex.Pattern; /** * Run the input through the model and see if it matches. * * The output value is the generated label, the Pair is the expected label and true if they match: */ public class BayesTestMapper extends Mapper{ private static final Pattern SLASH = Pattern.compile("/"); private AbstractNaiveBayesClassifier classifier; @Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); Configuration conf = context.getConfiguration(); Path modelPath = HadoopUtil.getSingleCachedFile(conf); NaiveBayesModel model = NaiveBayesModel.materialize(modelPath, conf); boolean isComplementary = Boolean.parseBoolean(conf.get(TestNaiveBayesDriver.COMPLEMENTARY)); // ensure that if we are testing in complementary mode, the model has been // trained complementary. a complementarty model will work for standard classification // a standard model will not work for complementary classification if (isComplementary) { Preconditions.checkArgument((model.isComplemtary()), "Complementary mode in model is different than test mode"); } if (isComplementary) { classifier = new ComplementaryNaiveBayesClassifier(model); } else { classifier = new StandardNaiveBayesClassifier(model); } } @Override protected void map(Text key, VectorWritable value, Context context) throws IOException, InterruptedException { Vector result = classifier.classifyFull(value.get()); //the key is the expected value context.write(new Text(SLASH.split(key.toString())[1]), new VectorWritable(result)); } }
分享文章:mahout0.8入门
浏览路径:http://myzitong.com/article/pjcjjg.html