Hadoop云计算平台搭建方案2.1(20200515180949)

HadoopHadoop4Hadoop1master3slaveLinuxVMwareWorkstationVMwareWorkstationJDKJavamasterJDKHadoopmasterslave1slave2ZookeeperHBaseHadoopHadoop4HadoopVMwareWorkstation44VMwareWorkstationVMwareWorkstation4PCVMwareWorkstationmasterLinuxUbuntumasterPCLinuxhadoopterminalUbuntuWindowsVMareToolsUbuntuVM(M)VMareTools(T)VMareToolsVMareTools/home/hadooptarxzvf/home/hadoop/vmware-tools-distribterminalcd/home/hadoop/vmware-tools-distrib$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATHexportPATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATHocal/share/',butfailed:Nosuchfileordirectory(gedit:2949):Gtk-WARNING**:Attemptingtostorechangesinto`/root/.local/share/',butfailed:Failedtocreatefile'/root/.local/share/':Nosuchfileordirectory(gedit:2949):Gtk-WARNING**:Attemptingtosetthepermissionsof`/root/.local/share/',butfailed:Nosuchfileordirectorysudomkdir-p/root/.local/share/ubuntualt+ctrl+F2ubunturoot/usr/bin/vi/etc/profileetc/profilehadoopdeleteIPmasterNamenodeslave1Datanodeslave2Datanodeslave3Datanode:wqctrl+alt+delhadoophadooproot/home/hadoop/dfstmpdfsnamedatamkdir/home/hadoop/dfs/name/description/propertypropertynamevalue*/value/propertypropertynamevalue*/value/property/configuration4configurationpropertynamevalue/propertypropertynamevaluefile:/home/hadoop/dfs/name/value/propertypropertynamevaluefile:/home/hadoop/dfs/data/value/propertypropertyname/namevalue4/value/propertypropertynamevaluetrue/value/property/configurationHDFS3datanode33HDFS5configurationpropertynamevalueyarn/value/propertypropertynamevalue/propertypropertynamevalue/property/configuration6/configuration!--SitespecificYARNconfigurationproperties--propertynamevaluemapreduce_shuffle/value/propertypropertynamevalue/propertypropertynamevalue/propertypropertynamevalue/propertypropertynamevalue/propertypropertynamevalue/propertypropertynamevalue/property/configuration7#exportJAVA_HOME=/home/y/libexec/exportJAVA_HOME=/home/hadoop/HadoopIPHadoopSSHSSHHadoopSSHSSHsudoapt-getupdateterminalopenssh-serversudoapt-getinstallopenssh-serveropenssh-serverwhichssh/usr/bin/sshsshwhichsshd/usr/bin/sshdsshdopen-serverSlavemastermasterubuntumaster(B)masterslave1slave2slave3VMslave1masterPCVMwareslave1slave2slave3IPHadoopTCP/IPIPPCEditConnectionsNetworkConnectionsEditEditingEthernetconnection1IPv4SettingsMethodManualIPIPIPDNSserversIPAddressesDNSDNSserversmasterIPAddress(Netmask)Gatewaysalve1IPDNSIPslave2IPDNSIPslave3IPDNSIPubuntumasteretc/hostnamesudogedit/etc/hostnameetc/hostnamemasterslave1slave2slave3slave1slave1slave2slave2slave3slave3hostshostsIPmaster/etc/hostssudogedit/etc/hosts/etc/hostsmasterslave1slave2slave3slave1slave2slave3hostsmastermasterslave1slave2slave3SSHSSHHadoop4slave1slave2slave3mastermasterslaveslave1masterssh-keygen-trsash/()id_rsa2slave1maserterminalssh-keygen-trsash/()id_rsascp~/.ssh/masterscp~/.ssh/hadoop@master:/home/hadoop/.ssh/3slave2maserslave2slave1ssh-keygen-trsash/hadoop@master:/home/hadoop/.ssh/4slave3maserslave3slave1ssh-keygen-trsash/hadoop@master:/home/hadoop/.ssh/5masterslavemaster/home/hadoop/.ssh/mvauthorized_keysslaveauthorized_keyscatauthorized_keyscatauthorized_keyscatauthorized_keysauthorized_keysslavescpauthorized_keyshadoop@slave1:/home/hadoop/.ssh/authorized_keysscpauthorized_keyshadoop@slave2:/home/hadoop/.ssh/authorized_keysscpauthorized_keyshadoop@slave3:/home/hadoop/.ssh/authorized_keys6sshslave1Agentadmittedfailuretosignusingthekeyslave1ssh-agentssh-agentid_rsassh-agentssh-addid_rsaHadoopHDFS1namenodeHadoopmasterHDFShadoop/home/hadoop/Namenodecd/home/hadoop/namenodeformatStoragedirectory/home/hadoop/hdfs/namehasbeensuccessfullyformattedHDFS/home/hdoop/dfs/name/2HDFShadoopnamenodestartnamenodedatanodestartdatanodenamenodedatanodejpsjpshadoop3588NameNode3692DataNode3757JpsHFShadoop/home/hadoop/cd/home/hadoop/./HDFS./jps4177SecondaryNameNode3588NameNode3692DataNode4285JpsmasterSecondaryNameNodeNameNodeslaveDataNodeYARNhadoop/home/hadoop/cd/home/hadoop/./YARNterminal./jpsmaster4177SecondaryNameNode4660Jps3588NameNode4411ResourceManager3692DataNodemasterResourceManagerNameNodeSecondaryNameNodejpsslave3329DataNode3670Jps3484NodeManagerslaveDataNodeNodeManager2Hadoop./././/home/hadoop/dfs/home/hadoop/dfsrm-rf/home/hadoop/dfs/rm-rf/home/hadoop/tmphdfsnamenodeformat3slaveDataNodeasudoufwdisablebhostsIPhadoopIPcformatnamenodenamenodedatanodeclusterIDslaveClusterIDmasterdatanodeCID(home/hadoop/dfs/data/current/VERSION)/home/hadoop/dfs/data/currenHadoopHadoopHadoopHadoopWordCountWordCountHadoopMapReduceHDFSWordCountHadoop1HDFS/home/hadoop/inputinputmkdir/home/hadoop/inputcd/home/hadoop/inputfile01file02echohelloworldbyeworldfile01ar,${}/lib,${}/lib/*.jar,${}/../lib/*.jar,/usr/lib/hadoop/*.jar,/home/hadoop/2/home/hadoop/=/home/hadoop/$HADOOP_HOMEjar1OraclejdbcOracleHBaseHDFSHadoopOracleSqoopOraclejdbcjarSqoop/home/hadoop/