hadoop使用简介-编程知识

hadoop使用简介

git clone

hadoop源码地址：https://gitee.com/CHNnoodle/hadoop.git
git clone错误：

Filename too long错误，使用git config --global core.longpaths true
git clone https://gitee.com/CHNnoodle/hadoop.git -b rel/release-3.2.2 拉取指定tag版本

hadoop安装包地址：https://mirrors.cloud.tencent.com/apache/hadoop/common/

windows平台下载，https://github.com/cdarlint/winutils替换hadoop对应版本的bin文件

hadoop

core-site.xml，hadoop中间文件的输出路径

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration><property><name>hadoop.tmp.dir</name><value>/D:/ruanjian/hadoop/hadoop-3.3.5/data</value>    </property><property><name>fs.defaultFS</name><value>hdfs://localhost:9000</value></property>
</configuration>

hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration><property><name>dfs.replication</name><value>1</value></property><property><name>dfs.namenode.name.dir</name><value>/D:/ruanjian/hadoop/hadoop-3.3.5/data/namenode</value></property><property><name>dfs.datanode.data.dir</name><value>/D:/ruanjian/hadoop/hadoop-3.3.5/data/datanode</value></property>
</configuration>

mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration><property><name>mapreduce.framework.name</name><value>yarn</value></property>
</configuration>

yarn-site.xml

<?xml version="1.0"?>
<configuration><property><name>yarn.nodemanager.aux-services</name><value>mapreduce_shuffle</value></property><property><name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name><value>org.apache.hahoop.mapred.ShuffleHandler</value></property>
</configuration>