hadoop服务部署指南

hadoop官方网站Apache Hadoop下载Index of /hadoop/common/hadoop-3.3.6部署serve1[rootserver1 ~]# useradd hadoop[rootserver1 ~]# passwd hadoop[rootserver1 ~]# mv hadoop-3.3.6.tar.gz jdk-8u181-linux-x64.tar.gz /home/hadoop[rootserver1 ~]# su - hadoop[hadoopserver1 ~]$ tar zxf hadoop-3.3.6.tar.gz[hadoopserver1 ~]$ tar zxf jdk-8u181-linux-x64.tar.g[hadoopserver1 ~]$ ln -s hadoop-3.3.6 hadoop[hadoopserver1 ~]$ ln -s jdk1.8.0_181 jdk[hadoopserver1 ~]$ cd jdk/bin/[hadoopserver1 bin]$ pwd/home/hadoop/jdk/bin[hadoopserver1 bin]$ cd[hadoopserver1 ~]$ vim .bash_profile[hadoopserver1 ~]$ source .bash_profile[hadoopserver1 ~]$ cd hadoop/etc/hadoop/配置文件[hadoopserver1 hadoop]$ vim hadoop-env.shexport JAVA_HOME/home/hadoop/jdk[hadoopserver1 hadoop]$ cd ..[hadoopserver1 etc]$ cd ..[hadoopserver1 hadoop]$ bin/hadoop 打印出命令行表示可以工作单机模式[hadoopserver1 hadoop]$ mkdir input[hadoopserver1 hadoop]$ cp etc/hadoop/*.xml input[hadoopserver1 hadoop]$ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.6.jar[hadoopserver1 hadoop]$ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.6.jar grep input output dfs[a-z.][hadoopserver1 hadoop]$ cat output/*1 dfsadmin单机分布式运作[hadoopserver1 hadoop]$ cd etc/hadoop[hadoopserver1 hadoop]$ vim core-site.xmlconfigurationpropertynamefs.defaultFS/namevaluehdfs://localhost:9000/value/property/configuration[hadoopserver1 hadoop]$ vim hdfs-site.xmlconfigurationpropertynamedfs.replication/namevalue1/value/property/configuration免密[hadoopserver1 hadoop]$ ssh-keygen[hadoopserver1 hadoop]$ ssh-copy-id localhost[hadoopserver1 hadoop]$ ssh localhost[hadoopserver1 ~]$ logoutConnection to localhost closed. 注意退出连接[hadoopserver1 hadoop]$ cd ..[hadoopserver1 etc]$ cd ..[hadoopserver1 hadoop]$ bin/hdfs namenode -format[hadoopserver1 hadoop]$ ssh server1[hadoopserver1 ~]$ logout[hadoopserver1 hadoop]$ sbin/start-dfs.shStarting namenodes on [localhost]Starting datanodesStarting secondary namenodes [server1][hadoopserver1 hadoop]$ jps[hadoopserver1 hadoop]$ netstat -antlp 确认是java进程访问http://192.168.164.161:9870创建用户路径[hadoopserver1 hadoop]$ bin/hdfs dfs -mkdir /user访问Browsing HDFS[hadoopserver1 hadoop]$ bin/hdfs dfs -mkdir /user/hadoop扫描[hadoopserver1 hadoop]$ bin/hdfs dfs -ls[hadoopserver1 hadoop]$ bin/hdfs dfs -mkdir input 默认写入/user/hadoop[hadoopserver1 hadoop]$ bin/hdfs dfs -put etc/hadoop/*.xml input[hadoopserver1 hadoop]$ bin/hdfs dfs -put etc/hadoop/*.xml input[hadoopserver1 hadoop]$ bin/hdfs dfs -ls input[hadoopserver1 hadoop]$ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.6.jar[hadoopserver1 hadoop]$ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.6.jar wordcount input output[hadoopserver1 hadoop]$ cd output/[hadoopserver1 output]$ lspart-r-00000 _SUCCESS 本地删除操作系统不影响注意同名停止分布式文件系统sbin/stop-dfs.shjps确认停止3台虚拟机server1[hadoopserver1 hadoop]$ logoutnfs共享,20个节点以内[rootserver1 ~]# yum install -y nfs-utils[rootserver1 ~]# id hadoopuid1001(hadoop) gid1001(hadoop) groups1001(hadoop)[rootserver1 ~]# vim /etc/exports/home/hadoop *(rw,sync,anonuid1001,anongid1001)[rootserver1 ~]# systemctl enable --now nfs[rootserver1 ~]# showmount -eExport list for server1:/home/hadoop *[rootserver2 ~]# useradd hadoop[rootserver2 ~]# echo 123456 | passwd --stdin hadoop[rootserver3 ~]# useradd hadoop[rootserver3 ~]# echo 123456 | passwd --stdin hadoop[rootserver2 ~]# yum install -y nfs-utils[rootserver3 ~]# yum install -y nfs-utils[rootserver2 ~]# showmount -e server1Export list for server1:/home/hadoop *[rootserver2 ~]# mount 192.168.164.161:/home/hadoop/ /home/hadoop/[rootserver2 ~]# df[rootserver2 ~]# ll -d /home/hadoop/drwx------ 6 hadoop hadoop 262 Aug 22 09:30 /home/hadoop/[rootserver2 ~]# id hadoop[rootserver3 ~]# mount 192.168.164.161:/home/hadoop/ /home/hadoop/server1[rootserver1 ~]# su - hadoop[hadoopserver1 ~]$ ssh server1[hadoopserver1 ~]$ logout[hadoopserver1 ~]$ ssh server2[hadoopserver2 ~]$ logout[hadoopserver1 ~]$ ssh server3[hadoopserver3 ~]$ logout[hadoopserver1 ~]$ cd hadoop修改配置文件[hadoopserver1 hadoop]$ cd etc/hadoop/[hadoopserver1 hadoop]$ vim core-site.xmlconfigurationpropertynamefs.defaultFS/namevaluehdfs://server1:9000/value/property/configuration[hadoopserver1 hadoop]$ vim hdfs-site.xmlconfigurationpropertynamedfs.replication/namevalue2/value/property/configuration[hadoopserver1 hadoop]$ vim workersserver2server3重新格式化[hadoopserver1 hadoop]$ cd ..[hadoopserver1 etc]$ cd ..[hadoopserver1 hadoop]$ bin/hdfs namenode -format[hadoopserver1 hadoop]$ sbin/start-dfs.shStarting namenodes on [server1]Starting datanodesStarting secondary namenodes [server1][hadoopserver1 hadoop]$ jps5475 NameNode5700 SecondaryNameNode5820 Jps[hadoopserver1 hadoop]$ bin/hdfs dfs -mkdir /user[hadoopserver1 hadoop]$ bin/hdfs dfs -mkdir /user/hadoop[hadoopserver1 hadoop]$ bin/hdfs dfs -mkdir input[hadoopserver1 hadoop]$ bin/hdfs dfs -put etc/hadoop/*.xml input访问https://192.168.164.161:9870/explorer.html#/配置调度器与master不放在一起[hadoopserver1 hadoop]$ vim etc/hadoop/mapred-site.xmlconfigurationpropertynamemapreduce.framework.name/namevalueyarn/value/propertypropertynamemapreduce.application.classpath/namevalue$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*/value/property/configuration[hadoopserver1 hadoop]$ vim etc/hadoop/yarn-site.xmlconfigurationpropertynameyarn.nodemanager.aux-services/namevaluemapreduce_shuffle/value/propertypropertynameyarn.nodemanager.env-whitelist/namevalueJAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME/value/property/configuration[hadoopserver1 hadoop]$ sbin/start-yarn.sh[hadoopserver1 hadoop]$ jps另开一台虚拟机server4互相解析[rootserver4 ~]# yum install -y nfs-utils[rootserver4 ~]# useradd hadoop[rootserver4 ~]# echo 123456 | passwd --stdin hadoop[rootserver4 ~]# mount 192.168.164.161:/home/hadoop/ /home/hadoop/[rootserver4 ~]# df[rootserver4 ~]# ll -d /home/hadoopdrwx------ 6 hadoop hadoop 262 Aug 22 10:04 /home/hadoop 注意权限[hadoopserver1 ~]$ ssh server4[hadoopserver1 ~]$ logout[hadoopserver4 ~]$ cd hadoop[hadoopserver4 hadoop]$ vim etc/hadoop/workersserver2server3server4[hadoopserver4 hadoop]$ bin/hdfs --daemon start datanode[hadoopserver4 hadoop]$ bin/yarn --daemon start nodemanager[hadoopserver4 hadoop]$ jps1780 NodeManager1673 DataNode1871 Jps访问192.168.164.161:9870访问192.168.164.161:8088