S-JIS[2011-09-24/2013-10-05] �ύX����

Hadoop CDH3

CDH3�iCloudera's Distribution including Apache Hadoop v3�j�̃C���X�g�[���ɂ‚��āB


CDH�̊T�v

CDH�iCloudera's Distribution including Apache Hadoop�j�́AHadoop�̃f�B�X�g���r���[�V�����B[2011-09-25]
Hadoop��CDH�̊֌W�́A�Ⴆ��Linux��RedHat�̊֌W�Ɠ����B
�‚܂茳��Apache Hadoop���J�X�^�}�C�Y���i�p�b�`�𓖂āj�Ĕz�z���Ă���B

CDH�̒��ɂ�Hadoop�֘A�v���_�N�g�iHive�EPig��HBase�ESqoop���j���܂܂�A�����͌•ʂɃC���X�g�[������B
���������Ė{���́uCDH���C���X�g�[������v�ƌ����̂͐������Ȃ��āA�uCDH�ł�Hadoop���C���X�g�[������v�uCDH�ł�Hive���C���X�g�[������v�ƌ����̂��������B


Windows7�ւ̃C���X�g�[��

CDH3�̃A�[�J�C�u���_�E�����[�h���Ă���΁A���ʂ�Hadoop��Windows�ւ̃C���X�g�[���Ɠ��l�ɃC���X�g�[���ł���B[2011-11-16]

  1. CDH3�̃A�[�J�C�u���_�E�����[�h����B
  2. CDH3�̃A�[�J�C�u��W�J����B
    Cygwin�ňȉ��̃R�}���h�����s����B�i�g���q��tar.gz�Ȃ̂�gzip���k����Ă��邪�A�ŋ߂�tar�R�}���h�͒��ډ𓀂ł���͗l�j
    $ cd /usr/local
    $ tar xf $(cygpath -u "C:\Users\hishidama\Downloads\hadoop-0.20.2-cdh3u2.tar.gz")
  3. JDK1.6�ւ̃V���{���b�N�����N������Ă����B
    $ ln -s "/cygdrive/c/Program Files (x86)/Java/jdk1.6.0_27" /usr/local/jdk1.6
  4. Windows�̊‹��ϐ���ݒ肷��B
    HADOOP_HOME D:/cygwin/usr/local/hadoop-0.20.2-cdh3u2 �@
    Path �`;%HADOOP_HOME%\bin ������PATH�̐ݒ�̖����ɒlj��B
  5. HADOOP_HOME/bin/hadoop-config.sh���C������B�iHADOOP_HOME�̐ݒ��lj�����j
    # the root of the Hadoop installation
    if [ -z "$HADOOP_HOME" ]; then
      export HADOOP_HOME=`dirname "$this"`/..
    else
      export HADOOP_HOME=$(cygpath -u "$HADOOP_HOME")
    fi
  6. HADOOP_HOME/conf/hadoop-env.sh���C������B�iJAVA_HOME�̐ݒ��lj�����j
    # export JAVA_HOME=/usr/lib/j2sdk1.6-sun
    export JAVA_HOME=/usr/local/jdk1.6
  7. ������hadoop�R�}���h�����s���Ă݂�B
    $ hadoop
    $ hadoop classpath		�cWindows�Ƃ��Ẵp�X���\��������OK
    $ hadoop version  		�cCDH3�̃o�[�W�������\��������OK

��Hadoop0.20����������CDH3�‹��������@


CentOS�ւ̃C���X�g�[��

CentOS�ւ́Ayum�R�}���h���g�p���ăC���X�g�[�����邱�Ƃ��o����B[2011-09-25]

  1. JDK1.6���C���X�g�[���������B
  2. yum���|�W�g���[��Cloudera�̃T�C�g��lj�����B
    �ǂ�OS�̏ꍇ�ɂǂ�repo�t�@�C�����_�E�����[�h����΂悢���́ACDH Version and Packaging Information�ɍڂ��Ă���B[2012-05-10]
    CentOS5 http://archive.cloudera.com/redhat/cdh/cloudera-cdh3.repo
    CentOS6 http://archive.cloudera.com/redhat/6/x86_64/cdh/cloudera-cdh3.repo
    # cd /etc/yum.repos.d/
    # wget http://archive.cloudera.com/redhat/cdh/cloudera-cdh3.repo
    # yum update yum
  3. Hadoop�{�̂��_�E�����[�h�E�C���X�g�[������B�i���ꂾ���ŁA�P�Ɗ‹��͎��s�”\�j
    # yum install -y hadoop-0.20
    ���ꂪ����I������ƁA�����I��hdfs��mapred�Ƃ������[�U�[�������B
    �uhadoop�v�Ƃ������[�U�[���������ꍇ�́Ahdfs�Ƃ������[�U�[���ɕύX�����B�i�p�X���[�h��z�[���f�B���N�g���[�ɂ͕ύX�Ȃ��j
  4. �ꉞ����m�F���Ă����B
    $ hadoop
    $ hadoop fs -ls
    $ hadoop jar /usr/lib/hadoop/hadoop-*-examples.jar pi 4 2000	���~�������v�Z����T���v��

���Ȃ݂ɁA�uyum search hadoop�v��Hadoop�֘A�A�v���P�[�V�����̈ꗗ���\�������B
�uyum list installed | grep hadoop�v��Hadoop�֘A�̃C���X�g�[���ς݃A�v���P�[�V�����ꗗ���\�������B

�‹��ϐ�HADOOP_HOME���`����Ƃ��́A�u/usr/lib/hadoop�v���w�肷��B�i���������̂�/usr/lib/hadoop-0.20�ɂ���j
�R���t�B�O�t�@�C���́u/etc/hadoop/conf�v���w�肷��B�i���������̂�/etc/hadoop-0.20�ɂ���A�f�t�H���g�ł͂��̉���conf.empty���w���Ă���B�ʂ�conf�ւ̐؂�ւ��ɂ�alternatives�R�}���h���g�p����j


�\�[�X�̃_�E�����[�h

CDH3�̃A�[�J�C�u�̒���src�f�B���N�g���[����CDH3�̃\�[�X�������Ă���B[2011-11-16]

��Maven���g����CDH3�̃\�[�X���_�E�����[�h������@


yum�R�}���h���g���ă\�[�X���_�E�����[�h���邱�Ƃ��o����B[2012-01-21]

# yum -y install hadoop-0.20-source

���̏ꍇ�A�u/usr/src/hadoop-0.20�v�Ƀ\�[�X���C���X�g�[�������B


CDH3��core-site.xml��hdfs-site.xml�Emapred-site.xml�̃f�t�H���g�l�́A�ȉ��̃y�[�W�Ō��J����Ă���B[2012-03-16]


CentOS�[�����U�‹��̍\�z

CentOS��CDH3�̋[�����U�‹��������@�B[2012-03-16]

�ȉ��̃R�}���h�����s���ċ[�����U�‹��̐ݒ�t�@�C�����C���X�g�[������B

# yum -y install hadoop-0.20-conf-pseudo

�C���X�g�[�������ƁA/etc/hadoop-0.20/conf.pseudo���쐬�����B
�i�܂��AHDFS�̃t�H�[�}�b�g���s����j
CDH�ł�conf�̎��́iconf���w���ꏊ�j��alternatives�ɂ���Đ��䂳��Ă���̂ŁA�ȉ��̃R�}���h�Ŋm�F�ł���B

# alternatives --display hadoop-0.20-conf
hadoop-0.20-conf - �X�e�[�^�X�͎蓮�ł��B
�����N�͌��� /etc/hadoop-0.20/conf.empty ���w���Ă��܂��B
/etc/hadoop-0.20/conf.empty - �D�捀�� 10
/etc/hadoop-0.20/conf.pseudo - �D�捀�� 30
���݂́u�œK�v�o�[�W������ /etc/hadoop-0.20/conf.pseudo �ł��B

conf.empty�̓X�^���h�A���[�����[�h�p�̒�`�Ȃ̂ŁA�[�����U�‹��̒�`�ɐ؂�ւ��Ă��B

# alternatives --config hadoop-0.20-conf

2 �v���O���������� 'hadoop-0.20-conf' ��񋟂��܂��B

�I��         �R�}���h
-----------------------------------------------
 + 1           /etc/hadoop-0.20/conf.empty
*  2           /etc/hadoop-0.20/conf.pseudo

Enter �������Č��݂̑I�� [+] ��ێ����邩�A�I��ԍ�����͂��܂�:2

# alternatives --display hadoop-0.20-conf
hadoop-0.20-conf - �X�e�[�^�X�͎蓮�ł��B
�����N�͌��� /etc/hadoop-0.20/conf.pseudo ���w���Ă��܂��B
/etc/hadoop-0.20/conf.empty - �D�捀�� 10
/etc/hadoop-0.20/conf.pseudo - �D�捀�� 30
���݂́u�œK�v�o�[�W������ /etc/hadoop-0.20/conf.pseudo �ł��B

����ŁA/etc/hadoop/conf�i�V���{���b�N�����N�j��/etc/hadoop-0.20/conf.pseudo���w���悤�ɂȂ�B


hadoop-0.20-conf-pseudo���C���X�g�[������ƁAHadoop���N������ׂ̃V�F���X�N���v�g�iCDH�p�j���ꏏ�ɃC���X�g�[�������B

�‚܂�A�ȉ��̃C���X�g�[�����•ʂɍs���K�v�͖����B

yum install hadoop-0.20-namenode
yum install hadoop-0.20-secondarynamenode
yum install hadoop-0.20-jobtracker
yum install hadoop-0.20-datanode
yum install hadoop-0.20-tasktracker

���������āA�ȉ��̃R�}���h��Hadoop�̊e�f�[�������N���ł���B
�i�[�����U�‹��Ȃ̂ŁA1�‚̃}�V�����NameNode�EJobTracker�EDataNode�ETaskTracker���N������B
 SecondaryNameNode��Hadoop�����p���邾���Ȃ�L���Ă������Ă��֌W�Ȃ��j

# /etc/init.d/hadoop-0.20-namenode start
# /etc/init.d/hadoop-0.20-jobtracker start
# /etc/init.d/hadoop-0.20-datanode start
# /etc/init.d/hadoop-0.20-tasktracker start

��start�̑����stop�ɂ���ƁA��~

�e�f�[�����̋N���Łu[OK]�v�Əo�Ă��A�ݒ肪���������ꍇ�͒���ɗ����Ă��鎖������̂�(���)�A�N�����Ă��邩�ǂ������m�F����B

# /etc/init.d/hadoop-0.20-namenode status
# /etc/init.d/hadoop-0.20-jobtracker status
# /etc/init.d/hadoop-0.20-datanode status
# /etc/init.d/hadoop-0.20-tasktracker status
# jps
20849 Jps
20564 DataNode
20701 TaskTracker
20380 NameNode
20491 JobTracker

�܂��AOS�̋N�����Ɋe�f�[�����������I�ɋN������悤�ݒ肳���B

# chkconfig --list | grep hadoop
hadoop-0.20-datanode    0:off   1:off   2:off   3:on    4:off   5:on    6:off
hadoop-0.20-jobtracker  0:off   1:off   2:off   3:on    4:off   5:on    6:off
hadoop-0.20-namenode    0:off   1:off   2:off   3:on    4:off   5:on    6:off
hadoop-0.20-secondarynamenode   0:off   1:off   2:off   3:on    4:off   5:on   6:off
hadoop-0.20-tasktracker 0:off   1:off   2:off   3:on    4:off   5:on    6:off

�i������run���x���ŁA3��GUI�����OS���N������ꍇ�A5��GUI�����ŋN������ꍇ�̐ݒ�炵���j
�ion����OS�̋N�����ɂ��̃f�[�������N�������j

OS�̋N�����Ƀf�[�������N�����Ȃ��悤�ɂ���ɂ́A�ݒ��off�ɂ��Ă��΂����B

# chkconfig hadoop-0.20-namenode off
# chkconfig hadoop-0.20-secondarynamenode off
# chkconfig hadoop-0.20-jobtracker off
# chkconfig hadoop-0.20-datanode off
# chkconfig hadoop-0.20-tasktracker off

# chkconfig --list | grep hadoop
hadoop-0.20-datanode    0:off   1:off   2:off   3:off   4:off   5:off   6:off
hadoop-0.20-jobtracker  0:off   1:off   2:off   3:off   4:off   5:off   6:off
hadoop-0.20-namenode    0:off   1:off   2:off   3:off   4:off   5:off   6:off
hadoop-0.20-secondarynamenode   0:off   1:off   2:off   3:off   4:off   5:off  6:off
hadoop-0.20-tasktracker 0:off   1:off   2:off   3:off   4:off   5:off   6:off

CentOS�ւ̓���o�[�W�����̃C���X�g�[��

��L�̕��@�i�f�t�H���g�j�ł́ACDH�̍ŐV�ł��C���X�g�[������邱�ƂɂȂ�B[2012-03-31]
�܂��ACentOS�iyum�j�̎����A�b�v�f�[�g��CDH���ŐV�łɍX�V����邱�ƂɂȂ�B

����̃o�[�W�������C���X�g�[���������i����ȏ�A�b�v�f�[�g���������Ȃ��j�ꍇ�́Arepo�t�@�C�����Ƀo�[�W�������w�肷��B

/etc/yum.repos.d/cloudera-cdh3.repo�F

mirrorlist=http://archive.cloudera.com/redhat/cdh/3/mirrors
					��
mirrorlist=http://archive.cloudera.com/redhat/cdh/3u2/mirrors

������ʂ�yum install���s�������I

�Q�l�F �n�������WEB+DB Press vol.67 �⑫ (�NjL)


���łւ̃_�E���O���[�h

��U�ŐV�łŃC���X�g�[��������ɌÂ��o�[�W�����ɂ������ꍇ�Acloudera-cdh3.repo�����������Ă���ăC���X�g�[������B[2012-03-31]

�������A�ăC���X�g�[���R�}���h�uyum reinstall�v�����s���Ă��G���[�ɂȂ��Ă��܂��B

# vi /etc/yum.repos.d/cloudera-cdh3.repo	������3u1�ɏ�������
# yum reinstall hadoop-0.20
�`
================================================================================
 Package          Arch        Version                  Repository          Size
================================================================================
Reinstalling:
 hadoop-0.20      noarch      0.20.2+923.197-1         cloudera-cdh3       30 M

Transaction Summary
================================================================================
Remove        0 Package(s)
Reinstall     1 Package(s)
Downgrade     0 Package(s)

Total download size: 30 M
Is this ok [y/N]: y
Downloading Packages:
http://archive.cloudera.com/redhat/cdh/3u1/RPMS/noarch/hadoop-0.20-0.20.2%2B923.197-1.noarch.rpm: [Errno 14] HTTP Error 404: Not Found
Trying other mirror.


Error Downloading Packages:
  hadoop-0.20-0.20.2+923.197-1.noarch: failure: RPMS/noarch/hadoop-0.20-0.20.2+923.197-1.noarch.rpm from cloudera-cdh3: [Errno 256] No more mirrors to try.

�g�C���X�g�[���Ώۃo�[�W�������u0.20.2+923.197-1�v�����A�uhadoop-0.20-0.20.2%2B923.197-1.noarch.rpm�v�i%2B�́u+�v�̂��Ɓj�����‚���Ȃ��h�Ƃ����G���[�炵���B
��������0.20.2+923.197-1��3u3�̃o�[�W�����Ȃ̂ŁA3u1�̃f�B���N�g���[���ɖ����͓̂��R�ł��ȁO�O�G

�o�[�W�������Ɋւ��Ă�yum�̃L���b�V���Ɏc���Ă���炵���A�L���b�V�����N���A���Ă��K�v������B

# yum clean metadata

# yum reinstall hadoop-0.20
�`
Installed package hadoop-0.20-0.20.2+923.197-1.noarch not available.

�ǂ���reinstall�ł͏o���Ȃ��悤��(���)


�f���ɍ폜���ăC���X�g�[���������Əo����B

# vi /etc/yum.repos.d/cloudera-cdh3.repo	�����̃o�[�W��������������
# yum clean metadata
# yum remove hadoop-0.20
# yum install -y hadoop-0.20

����o�[�W�����ւ̃A�b�v�O���[�h

�t�ɁA�V�����o�[�W�����ւ̐؂�ւ��͓��ɖ��Ȃ��o����B[2012-03-31]

# vi /etc/yum.repos.d/cloudera-cdh3.repo	������3u1����3u2�ɏ�������
# yum upgrade hadoop-0.20

Hadoop�ڎ��֖߂� / �Z�p�����֖߂�
���[���̑��M��F�Ђ�����

�@

�@

�@

�@

�@

�@

�@

�@

�@

�@

�@

�@

�@

�@

�@

�@