hadoop hdfs-site.xml的配置

<?xml version="1.0"encoding="UTF-8"?>

<?xml-stylesheet type="text/xsl"href="configuration.xsl"?>

<configuration>

              <!--指定hdfs的nameservice为shcluster，需要和core-site.xml中的保持一致 -->

              <property>

                            <name>dfs.nameservices</name>

                            <value>shcluster</value>

              </property>

              <!—指定磁盘预留多少空间，防止磁盘被撑满用完，单位为bytes-->

              <property>

                            <name>dfs.datanode.du.reserved</name>

                            <value>107374182400</value>

              </property>

              <!--shcluster下面有两个NameNode，分别是namenode1，namenode2 -->

              <property>

                            <name>dfs.ha.namenodes.shcluster</name>

                            <value>namenode1,namenode2</value>

              </property>

              <!--namenode1的RPC通信地址,这里端口要和core-site.xml中fs.defaultFS保持一致 -->

              <property>

                            <name>dfs.namenode.rpc-address.shcluster.namenode1</name>

                            <value>app1:9000</value>

              </property>

              <!--namenode1的http通信地址 -->

              <property>

                            <name>dfs.namenode.http-address.shcluster.namenode1</name>

                            <value>app1:50070</value>

              </property>

              <!--namenode2的RPC通信地址,这里端口要和core-site.xml中fs.defaultFS保持一致 -->

              <property>

                            <name>dfs.namenode.rpc-address.shcluster.namenode2</name>

                            <value>app2:9000</value>

              </property>

              <!--namenode2的http通信地址 -->

              <property>

                            <name>dfs.namenode.http-address.shcluster.namenode2</name>

                            <value>app2:50070</value>

              </property>

               <!--指定NameNode的元数据在JournalNode上的存放位置 -->

              <property>

                           <name>dfs.namenode.shared.edits.dir</name>                                               <value>qjournal://app1:8485;app2:8485;app3:8485:8485/shcluster</value>

              </property>

               <!--配置失败自动切换实现方式-->

              <property>

                            <name>dfs.client.failover.proxy.provider.shcluster</name>                          <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

              </property>

               <!--配置隔离机制,主要用户远程管理监听其他机器相关服务-->

              <property>

                            <name>dfs.ha.fencing.methods</name>

                            <value>sshfence</value>

              </property>

              <!--使用隔离机制时需要ssh免密码登陆 -->

              <property>

                            <name>dfs.ha.fencing.ssh.private-key-files</name>

                            <value>/home/hadoop/.ssh/id_rsa</value>

              </property>

               <!--指定NameNode的元数据在JournalNode上的存放位置 -->

              <property>

                            <name>dfs.journalnode.edits.dir</name>

                            <value>/home/hadoop/hadoop-2.7.4/journal </value>

              </property>

               <!--指定支持高可用自动切换机制-->

              <property>

                            <name>dfs.ha.automatic-failover.enabled</name>

                            <value>true</value>

              </property>

               <!--指定namenode名称空间的存储地址-->

              <property>

                            <name>dfs.namenode.name.dir</name>   

<value>file:/data1/hadoop/dfs/name,file:/data2/hadoop/dfs/name,file:/data3/hadoop/dfs/name,file:/data4/hadoop/dfs/name,file:/data5/hadoop/dfs/name,file:/data6/hadoop/dfs/name,file:/data7/hadoop/dfs/name,file:/data8/hadoop/dfs/name,file:/data9/hadoop/dfs/name,file:/data10/hadoop/dfs/name,file:/data11/hadoop/dfs/name,file:/data12/hadoop/dfs/name</value>>

              </property>

               <!--指定datanode数据存储地址-->

              <property>

                            <name>dfs.datanode.data.dir</name>

<value>file:/data1/hadoop/dfs/data,file:/data2/hadoop/dfs/data,file:/data3/hadoop/dfs/data,file:/data4/hadoop/dfs/data,file:/data5/hadoop/dfs/data,file:/data6/hadoop/dfs/data,file:/data7/hadoop/dfs/data,file:/data8/hadoop/dfs/data,file:/data9/hadoop/dfs/data,file:/data10/hadoop/dfs/data,file:/data11/hadoop/dfs/data,file:/data12/hadoop/dfs/data</value>              </property>

               <!--指定数据冗余份数-->

              <property>

                            <name>dfs.replication</name>

                            <value>3</value>

              </property>

               <!--指定可以通过web访问hdfs目录-->

              <property>

                            <name>dfs.webhdfs.enabled</name>

                            <value>false</value>

              </property>

               <property>

                            <name>ha.zookeeper.quorum</name>                                                         <value>app138:2181,app2:2181,app3:2181</value>

              </property>

               <property>

                            <name>dfs.namenode.handler.count</name>

                            <value>600</value>

                            <description>Thenumber of server threads for the namenode.</description>

              </property>

              <property>

                            <name>dfs.datanode.handler.count</name>

                            <value>600</value>

                            <description>Thenumber of server threads for the datanode.</description>

              </property>

              <property>

                            <name>dfs.client.socket-timeout</name>

                            <value>600000</value>

              </property>

              <property> 

                            <!--这里设置Hadoop允许打开最大文件数，默认4096，不设置的话会提示xcievers exceeded错误--> 

                            <name>dfs.datanode.max.transfer.threads</name> 

                            <value>409600</value> 

              </property> 

 <!--journal timeout setting-->

              <property>

               <name>dfs.qjournal.start-segment.timeout.ms</name>

               <value>600000</value>

       </property>

       <property>

               <name>dfs.qjournal.prepare-recovery.timeout.ms</name>

                <value>600000</value>

       </property>

       <property>

               <name>dfs.qjournal.accept-recovery.timeout.ms</name>

               <value>600000</value>

       </property>

       <property>

               <name>dfs.qjournal.finalize-segment.timeout.ms</name>

               <value>600000</value>

       </property>

       <property>

               <name>dfs.qjournal.select-input-streams.timeout.ms</name>

               <value>600000</value>

       </property>

       <property>

               <name>dfs.qjournal.get-journal-state.timeout.ms</name>

               <value>600000</value>

       </property>

       <property>

               <name>dfs.qjournal.new-epoch.timeout.ms</name>

               <value>600000</value>

       </property>

       <property>

               <name>dfs.qjournal.write-txns.timeout.ms</name>

               <value>600000</value>

       </property>

<!—start-balancer时，hdfs移动数据的速度，默认值为1M/S的速度。一般情况下设置为10M；设置的过大会影响当前job的运行-->

       <property>

               <name>dfs.datanode.balance.bandwidthPerSec</name>

               <value>10485760</value>

       </property>

 </configuration>

 </configuration>