{"id":356,"date":"2013-07-31T10:49:41","date_gmt":"2013-07-31T02:49:41","guid":{"rendered":"https:\/\/yanjingang.com\/blog\/?p=356"},"modified":"2016-01-28T10:51:03","modified_gmt":"2016-01-28T02:51:03","slug":"hadoop%e9%9b%86%e7%be%a4%e9%83%a8%e7%bd%b2%ef%bc%88%e9%83%a8%e7%bd%b2%e8%84%9a%e6%9c%ac%ef%bc%8cnamenode%e9%ab%98%e5%8f%af%e7%94%a8%ef%bc%89","status":"publish","type":"post","link":"https:\/\/yanjingang.com\/blog\/?p=356","title":{"rendered":"Hadoop\u96c6\u7fa4\u90e8\u7f72\uff08\u90e8\u7f72\u811a\u672c\uff0cnamenode\u9ad8\u53ef\u7528\uff09"},"content":{"rendered":"<p><strong>\u524d\u8a00<\/strong><\/p>\n<p>\u6298\u817e\u4e86\u4e00\u6bb5\u65f6\u95f4hadoop\u7684\u90e8\u7f72\u7ba1\u7406\uff0c\u5199\u4e0b\u6b64\u7cfb\u5217\u535a\u5ba2\u8bb0\u5f55\u4e00\u4e0b\u3002<\/p>\n<p>\u4e3a\u4e86\u907f\u514d\u5404\u4f4d\u505a\u90e8\u7f72\u8fd9\u79cd\u91cd\u590d\u6027\u7684\u52b3\u52a8\uff0c\u6211\u5df2\u7ecf\u628a\u90e8\u7f72\u7684\u6b65\u9aa4\u5199\u6210\u811a\u672c\uff0c\u5404\u4f4d\u53ea\u9700\u8981\u6309\u7740\u672c\u6587\u628a\u811a\u672c\u6267\u884c\u5b8c\uff0c\u6574\u4e2a\u73af\u5883\u57fa\u672c\u5c31\u90e8\u7f72\u5b8c\u4e86\u3002\u90e8\u7f72\u7684\u811a\u672c\u6211\u653e\u5728\u4e86\u5f00\u6e90\u4e2d\u56fd\u7684git\u4ed3\u5e93\u91cc(<a href=\"http:\/\/git.oschina.net\/snake1361222\/hadoop_scripts\" target=\"_blank\">http:\/\/git.oschina.net\/snake1361222\/hadoop_scripts<\/a>)\u3002<\/p>\n<p>\u672c\u6587\u7684\u6240\u6709\u90e8\u7f72\u90fd\u57fa\u4e8ecloudera\u516c\u53f8\u7684CDH4,CDH4\u662fcloudera\u516c\u53f8\u5305\u88c5\u597d\u7684hadoop\u751f\u6001\u5708\u4e00\u7cfb\u5217yum\u5305\uff0c\u628aCDH4\u653e\u5230\u81ea\u5df1\u7684yum\u4ed3\u5e93\u4e2d\uff0c\u80fd\u6781\u5927\u7684\u63d0\u9ad8hadoop\u73af\u5883\u90e8\u7f72\u7684\u7b80\u6613\u6027\u3002<\/p>\n<p>\u672c\u6587\u7684\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u6db5\u76d6\u4e86namenode\u7684HA\u5b9e\u73b0\uff0chadoop\u7ba1\u7406\u7684\u89e3\u51b3\u65b9\u6848\uff08hadoop\u914d\u7f6e\u6587\u4ef6\u7684\u540c\u6b65\uff0c\u5feb\u901f\u90e8\u7f72\u811a\u672c\u7b49\uff09\u3002<\/p>\n<p><strong>\u73af\u5883\u51c6\u5907<\/strong><\/p>\n<p>\u4e00\u5171\u75285\u53f0\u673a\u5668\u4f5c\u4e3a\u786c\u4ef6\u73af\u5883\uff0c\u5168\u90fd\u662fcentos 6.4<\/p>\n<ul>\n<li>namenode &amp; resourcemanager \u4e3b\u670d\u52a1\u5668: 192.168.1.1<\/li>\n<li>namenode &amp; resourcemanager \u5907\u670d\u52a1\u5668: 192.168.1.2<\/li>\n<li>datanode &amp; nodemanager \u670d\u52a1\u5668: 192.168.1.100 192.168.1.101 192.168.1.102<\/li>\n<li>zookeeper \u670d\u52a1\u5668\u96c6\u7fa4(\u7528\u4e8enamenode \u9ad8\u53ef\u7528\u7684\u81ea\u52a8\u5207\u6362): 192.168.1.100 192.168.1.101<\/li>\n<li>jobhistory \u670d\u52a1\u5668(\u7528\u4e8e\u8bb0\u5f55mapreduce\u7684\u65e5\u5fd7): 192.168.1.1<\/li>\n<li>\u7528\u4e8enamenode HA\u7684NFS: 192.168.1.100<\/li>\n<\/ul>\n<p>\u73af\u5883\u90e8\u7f72<\/p>\n<p><strong>\u4e00\u3001\u52a0\u5165CDH4\u7684YUM\u4ed3\u5e93<\/strong><\/p>\n<p>1.\u6700\u597d\u7684\u529e\u6cd5\u662f\u628acdh4\u7684\u5305\u653e\u5230\u81ea\u5efa\u7684yum\u4ed3\u5e93\u4e2d,\u5982\u4f55\u81ea\u5efayum\u4ed3\u5e93\u8bf7\u770b \u81ea\u5efaYUM\u4ed3\u5e93<\/p>\n<p>2.\u5982\u679c\u4e0d\u60f3\u81ea\u5efayum\u4ed3\u5e93\uff0c\u5728\u6240\u6709\u7684hadoop\u673a\u5668\u6267\u884c\u4ee5\u4e0b\u64cd\u4f5c\u52a0\u5165cdn4\u7684yum\u4ed3\u5e93<\/p>\n<ol>\n<li>wget\u00a0http:\/\/archive.cloudera.com\/cdh4\/one-click-install\/redhat\/6\/x86_64\/cloudera-cdh-4-0.x86_64.rpm<\/li>\n<li>sudo\u00a0yum\u00a0&#8211;nogpgcheck\u00a0localinstall\u00a0cloudera-cdh-4-0.x86_64.rpm<\/li>\n<\/ol>\n<p><strong>\u4e8c\u3001\u521b\u5efa\u7528\u4e8enamenode HA\u7684NFS\u670d\u52a1\u5668<\/strong><\/p>\n<p>1.\u767b\u5f55192.168.1.100,\u6267\u884c\u4ee5\u4e0b\u811a\u672ccreateNFS.sh<\/p>\n<ol>\n<li>#!\/bin\/bash<\/li>\n<li>yum\u00a0-y\u00a0install\u00a0rpc-bind\u00a0nfs-utils<\/li>\n<li>mkdir\u00a0-p\u00a0\/data\/nn_ha\/<\/li>\n<li>echo\u00a0&#8220;\/data\/nn_ha\u00a0\u00a0*(rw,root_squash,all_squash,sync)&#8221;\u00a0&gt;&gt;\u00a0\/etc\/exports<\/li>\n<li>\/etc\/init.d\/rpcbind\u00a0start<\/li>\n<li>\/etc\/init.d\/nfs\u00a0\u00a0start<\/li>\n<li>chkconfig\u00a0\u00a0&#8211;level\u00a0234\u00a0rpcbind\u00a0\u00a0\u00a0on<\/li>\n<li>chkconfig\u00a0\u00a0-level\u00a0234\u00a0nfs\u00a0\u00a0on<\/li>\n<\/ol>\n<p><a target=\"_blank\" name=\"i0\"><\/a><\/p>\n<h2>\u4e09\u3001Hadoop Namenode &amp; resourcemanager \u4e3b\u670d\u52a1\u5668 \u73af\u5883\u90e8\u7f72<\/h2>\n<p>1.\u767b\u5f55192.168.1.1\uff0c\u521b\u5efa\u811a\u672c\u76ee\u5f55\uff0c\u628a\u811a\u672c\u4ecegit\u4ed3\u5e93\u590d\u5236\u4e0b\u6765<\/p>\n<ol>\n<li>yum\u00a0\u2013y\u00a0install\u00a0git<\/li>\n<li>mkdir\u00a0\u2013p\u00a0\/opt\/<\/li>\n<li>cd\u00a0\/opt\/<\/li>\n<li>git\u00a0clone\u00a0http:\/\/git.oschina.net\/snake1361222\/hadoop_scripts.git<\/li>\n<li>\/etc\/init.d\/iptables\u00a0stop<\/li>\n<\/ol>\n<p>2.\u4fee\u6539hostname<\/p>\n<ol>\n<li>sh\u00a0\/opt\/hadoop_scripts\/deploy\/AddHostname.sh<\/li>\n<\/ol>\n<p>3.\u4fee\u6539\u90e8\u7f72\u811a\u672c\u7684\u914d\u7f6e\u6587\u4ef6<\/p>\n<ol>\n<li>vim\u00a0\/opt\/kingsoft\/hadoop_scripts\/deploy\/config<\/li>\n<li>#\u6dfb\u52a0master\u670d\u52a1\u5668\u7684\u5730\u5740\uff0c\u4e5f\u5c31\u662fnamenode\u4e3b\u670d\u52a1\u5668<\/li>\n<li>master=&#8221;192.168.1.1&#8243;<\/li>\n<li>#\u6dfb\u52a0nfs\u670d\u52a1\u5668\u5730\u5740<\/li>\n<li>nfsserver=&#8221;192.168.1.100&#8243;<\/li>\n<\/ol>\n<p>4.\u7f16\u8f91hosts\u6587\u4ef6(\u6b64\u6587\u4ef6\u4f1a\u540c\u6b65\u5230hadoop\u96c6\u7fa4\u6240\u6709\u673a\u5668)<\/p>\n<ol>\n<li>vim\u00a0\/opt\/hadoop_scripts\/share_data\/resolv_host<\/li>\n<li>127.0.0.1\u00a0\u00a0\u00a0localhost\u00a0localhost.localdomain\u00a0localhost4\u00a0localhost4.localdomain4<\/li>\n<li>::1\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0localhost\u00a0localhost.localdomain\u00a0localhost6\u00a0localhost6.localdomain6<\/li>\n<li>192.168.1.1\u00a0nn.dg.hadoop.cn<\/li>\n<li>192.168.1.2\u00a0nn2.dg.hadoop.cn<\/li>\n<li>192.168.1.100\u00a0dn100.dg.hadoop.cn<\/li>\n<li>192.168.1.101\u00a0dn101.dg.hadoop.cn<\/li>\n<li>192.168.1.102\u00a0dn102.dg.hadoop.cn<\/li>\n<\/ol>\n<p>5.\u6267\u884c\u90e8\u7f72\u811a\u672cCreateNamenode.sh<\/p>\n<ol>\n<li>sh\u00a0\/opt\/hadoop_scripts\/deploy\/CreateNamenode.sh<\/li>\n<\/ol>\n<p>6.\u642d\u5efasaltstack master<\/p>\n<p>PS:\u7c7b\u4f3c\u4e8epuppet\u7684\u670d\u52a1\u5668\u7ba1\u7406\u5f00\u6e90\u5de5\u5177\uff0c\u6bd4\u8f83\u8f7b\u91cf\uff0c\u5728\u8fd9\u91cc\u7528\u4e8e\u7ba1\u7406hadoop\u96c6\u7fa4\uff0c\u8c03\u5ea6datanode\uff0c\u5173\u4e8esaltstack\u7684\u8be6\u7ec6\u8bf7\u770b\u00a0<a href=\"http:\/\/developer.51cto.com\/#\" target=\"_blank\">SaltStack\u90e8\u7f72\u4e0e\u4f7f\u7528<\/a><\/p>\n<p>a.\u5b89\u88c5<\/p>\n<ol>\n<li>yum\u00a0-y\u00a0install\u00a0salt\u00a0salt-master<\/li>\n<\/ol>\n<p>b.\u4fee\u6539\u914d\u7f6e\u6587\u4ef6`\/etc\/salt\/master`,\u4e0b\u9762\u6807\u5fd7\u7684\u662f\u9700\u8981\u4fee\u6539\u7684\u9879<\/p>\n<ol>\n<li>\u4fee\u6539\u76d1\u542cIP\uff1a<\/li>\n<li>interface:\u00a00.0.0.0<\/li>\n<li>\u591a\u7ebf\u7a0b\u6c60\uff1a<\/li>\n<li>worker_threads:\u00a05<\/li>\n<li>\u5f00\u542f\u4efb\u52a1\u7f13\u5b58\uff1a\uff08\u5b98\u65b9\u63cf\u53d9\u5f00\u542f\u7f13\u5b58\u80fd\u627f\u8f7d5000minion\uff09<\/li>\n<li>job_cache<\/li>\n<li>\u5f00\u542f\u81ea\u52a8\u8ba4\u8bc1\uff1a<\/li>\n<li>auto_accept:\u00a0True<\/li>\n<\/ol>\n<p>c.\u5f00\u542f\u670d\u52a1<\/p>\n<ol>\n<li>\/etc\/init.d\/salt-master\u00a0start<\/li>\n<li>chkconfig\u00a0\u00a0salt-master\u00a0on<\/li>\n<\/ol>\n<p>7.\u90e8\u7f72\u8fc7\u7a0b\u4e2d\u5df2\u7ecf\u628a\u6211\u7684sample\u914d\u7f6e\u590d\u5236\u8fc7\u53bb\u4e86\uff0c\u6240\u4ee5\u53ea\u9700\u8981\u4fee\u6539\u90e8\u5206\u914d\u7f6e\u6587\u4ef6<\/p>\n<p><tt>a. \/etc\/hadoop\/conf\/hdfs-site.xml (\u5176\u5b9e\u5c31\u662f\u6309\u5b9e\u9645\u4fee\u6539\u4e3b\u673a\u540d\u5730\u5740)<\/tt><\/p>\n<ol>\n<li>&lt;property&gt;<\/li>\n<li>\u00a0\u00a0&lt;name&gt;dfs.namenode.rpc-address.mycluster.ns1&lt;\/name&gt;<\/li>\n<li>\u00a0\u00a0&lt;value&gt;nn.dg.hadoop.cn:8020&lt;\/value&gt;<\/li>\n<li>\u00a0\u00a0&lt;description&gt;\u5b9a\u4e49ns1\u7684rpc\u5730\u5740&lt;\/description&gt;<\/li>\n<li>&lt;\/property&gt;<\/li>\n<li>&lt;property&gt;<\/li>\n<li>\u00a0\u00a0&lt;name&gt;dfs.namenode.rpc-address.mycluster.ns2&lt;\/name&gt;<\/li>\n<li>\u00a0\u00a0&lt;value&gt;nn2.dg.hadoop.cn:8020&lt;\/value&gt;<\/li>\n<li>\u00a0\u00a0&lt;description&gt;\u5b9a\u4e49ns2\u7684rpc\u5730\u5740&lt;\/description&gt;<\/li>\n<li>&lt;\/property&gt;<\/li>\n<li>&lt;property&gt;<\/li>\n<li>\u00a0\u00a0\u00a0\u00a0&lt;name&gt;ha.zookeeper.quorum&lt;\/name&gt;<\/li>\n<li>\u00a0\u00a0\u00a0\u00a0&lt;value&gt;dn100.dg.hadoop.cn:2181,dn101.dg.hadoop.cn:2181,dn102.dg.hadoop.cn:2181,&lt;\/value&gt;<\/li>\n<li>\u00a0\u00a0\u00a0\u00a0&lt;description&gt;\u6307\u5b9a\u7528\u4e8eHA\u7684ZooKeeper\u96c6\u7fa4\u673a\u5668\u5217\u8868&lt;\/description&gt;<\/li>\n<li>&lt;\/property&gt;<\/li>\n<\/ol>\n<p>b.\u00a0<tt>mapred-site.xml<\/tt><\/p>\n<ol>\n<li>&lt;property&gt;<\/li>\n<li>\u00a0&lt;name&gt;mapreduce.jobhistory.address&lt;\/name&gt;<\/li>\n<li>\u00a0&lt;value&gt;nn.dg.hadoop.cn:10020&lt;\/value&gt;<\/li>\n<li>&lt;\/property&gt;<\/li>\n<li>&lt;property&gt;<\/li>\n<li>\u00a0&lt;name&gt;mapreduce.jobhistory.webapp.address&lt;\/name&gt;<\/li>\n<li>\u00a0&lt;value&gt;nn.dg.hadoop.cn:19888&lt;\/value&gt;<\/li>\n<li>&lt;\/property&gt;<\/li>\n<\/ol>\n<p>c.\u00a0<tt>yarn-site.xml<\/tt><\/p>\n<ol>\n<li>property&gt;<\/li>\n<li>\u00a0\u00a0&lt;name&gt;yarn.resourcemanager.resource-tracker.address&lt;\/name&gt;<\/li>\n<li>\u00a0\u00a0&lt;value&gt;nn.dg.hadoop.cn:8031&lt;\/value&gt;<\/li>\n<li>&lt;\/property&gt;<\/li>\n<li>&lt;property&gt;<\/li>\n<li>\u00a0\u00a0&lt;name&gt;yarn.resourcemanager.address&lt;\/name&gt;<\/li>\n<li>\u00a0\u00a0&lt;value&gt;nn.dg.hadoop.cn:8032&lt;\/value&gt;<\/li>\n<li>&lt;\/property&gt;<\/li>\n<li>&lt;property&gt;<\/li>\n<li>\u00a0\u00a0&lt;name&gt;yarn.resourcemanager.scheduler.address&lt;\/name&gt;<\/li>\n<li>\u00a0\u00a0&lt;value&gt;nn.dg.hadoop.cn:8030&lt;\/value&gt;<\/li>\n<li>&lt;\/property&gt;<\/li>\n<li>&lt;property&gt;<\/li>\n<li>\u00a0\u00a0&lt;name&gt;yarn.resourcemanager.admin.address&lt;\/name&gt;<\/li>\n<li>\u00a0\u00a0&lt;value&gt;nn.dg.hadoop.cn:8033&lt;\/value&gt;<\/li>\n<li>&lt;\/property&gt;<\/li>\n<\/ol>\n<p><strong>\u4e09\u3001Hadoop Namenode &amp; resourcemanager \u5907\u670d\u52a1\u5668 \u73af\u5883\u90e8\u7f72<\/strong><\/p>\n<p>1.\u767b\u5f55192.168.1.2\uff0c\u521b\u5efa\u811a\u672c\u76ee\u5f55\uff0c\u4ece\u4e3b\u670d\u52a1\u5668\u628a\u811a\u672c\u540c\u6b65\u8fc7\u6765<\/p>\n<ol>\n<li>\/etc\/init.d\/iptables\u00a0stop<\/li>\n<li>mkdir\u00a0\u2013p\u00a0\/opt\/hadoop_scripts<\/li>\n<li>rsync\u00a0\u2013avz\u00a0192.168.1.1::hadoop_s\u00a0\u00a0\u00a0\/opt\/hadoop_scripts<\/li>\n<\/ol>\n<p>2.\u6267\u884c\u90e8\u7f72\u811a\u672cCreateNamenode.sh<\/p>\n<ol>\n<li>sh\u00a0\/opt\/hadoop_scripts\/deploy\/CreateNamenode.sh<\/li>\n<\/ol>\n<p>3.\u540c\u6b65hadoop\u914d\u7f6e\u6587\u4ef6<\/p>\n<ol>\n<li>rsync\u00a0\u2013avz\u00a0192.168.1.1::hadoop_conf\u00a0\u00a0\/etc\/hadoop\/conf<\/li>\n<\/ol>\n<p>4.\u90e8\u7f72saltstack\u5ba2\u6237\u7aef<\/p>\n<ol>\n<li>sh\u00a0\/opt\/hadoop_scripts\/deploy\/salt_minion.sh<\/li>\n<\/ol>\n<p><strong>\u56db\u3001zookeeper\u670d\u52a1\u5668\u96c6\u7fa4\u90e8\u7f72<\/strong><\/p>\n<p><strong>zookeeper\u662f\u4e00\u4e2a\u5f00\u6e90\u5206\u5e03\u5f0f\u670d\u52a1\uff0c\u5728\u8fd9\u91cc\u7528\u4e8enamenode \u7684auto fail over\u529f\u80fd\u3002<\/strong><\/p>\n<p>1.\u5b89\u88c5<\/p>\n<ol>\n<li>yum\u00a0install\u00a0zookeeper\u00a0zookeeper-server<\/li>\n<\/ol>\n<p>2.\u4fee\u6539\u914d\u7f6e\u6587\u4ef6<tt>\/etc\/zookeeper\/conf\/zoo.cfg<\/tt><\/p>\n<ol>\n<li>maxClientCnxns=50<\/li>\n<li>#\u00a0The\u00a0number\u00a0of\u00a0milliseconds\u00a0of\u00a0each\u00a0tick<\/li>\n<li>tickTime=2000<\/li>\n<li>#\u00a0The\u00a0number\u00a0of\u00a0ticks\u00a0that\u00a0the\u00a0initial<\/li>\n<li>#\u00a0synchronization\u00a0phase\u00a0can\u00a0take<\/li>\n<li>initLimit=10<\/li>\n<li>#\u00a0The\u00a0number\u00a0of\u00a0ticks\u00a0that\u00a0can\u00a0pass\u00a0between<\/li>\n<li>#\u00a0sending\u00a0a\u00a0request\u00a0and\u00a0getting\u00a0an\u00a0acknowledgement<\/li>\n<li>syncLimit=5<\/li>\n<li>#\u00a0the\u00a0directory\u00a0where\u00a0the\u00a0snapshot\u00a0is\u00a0stored.<\/li>\n<li>dataDir=\/var\/lib\/zookeeper<\/li>\n<li>#\u00a0the\u00a0port\u00a0at\u00a0which\u00a0the\u00a0clients\u00a0will\u00a0connect<\/li>\n<li>clientPort=2181<\/li>\n<li>#\u8fd9\u91cc\u6307\u5b9azookeeper\u96c6\u7fa4\u5185\u7684\u6240\u6709\u673a\u5668,\u6b64\u914d\u7f6e\u96c6\u7fa4\u5185\u673a\u5668\u90fd\u662f\u4e00\u6837\u7684<\/li>\n<li>server.1=dn100.dg.hadoop.cn\u00a0:2888:3888<\/li>\n<li>server.2=dn101.dg.hadoop.cn:2888:3888<\/li>\n<\/ol>\n<p>3.\u6307\u5b9a\u5f53\u524d\u673a\u5668\u7684id,\u5e76\u5f00\u542f\u670d\u52a1<\/p>\n<ol>\n<li>#\u8b6c\u5982\u5f53\u524d\u673a\u5668\u662f192.168.1.100(dn100.dg.hadoop.cn),\u5b83\u662fserver.1\uff0cid\u662f1\uff0cSO\uff1a<\/li>\n<li>echo\u00a0&#8220;1&#8221;\u00a0&gt;\u00a0\u00a0\/var\/lib\/zookeeper\/myid<\/li>\n<li>chown\u00a0-R\u00a0zookeeper.zookeeper\u00a0\/var\/lib\/zookeeper\/<\/li>\n<li>service\u00a0zookeeper-server\u00a0init<\/li>\n<li>\/etc\/init.d\/zookeeper-server\u00a0start<\/li>\n<li>chkconfig\u00a0zookeeper-server\u00a0on<\/li>\n<li>#\u5982\u6b64\u7c7b\u63a8\uff0c\u90e8\u7f72192.168.1.101<\/li>\n<\/ol>\n<p><strong>\u4e94\u3001datanode &amp; nodemanager \u670d\u52a1\u5668\u90e8\u7f72<\/strong><\/p>\n<p>1.\u767b\u5f55datanode\u673a\u5668,\u521b\u5efa\u811a\u672c\u76ee\u5f55\uff0c\u4ece\u4e3b\u670d\u52a1\u5668\u628a\u811a\u672c\u540c\u6b65\u8fc7\u6765<\/p>\n<ol>\n<li>\/etc\/init.d\/iptables\u00a0stop<\/li>\n<li>mkdir\u00a0\u2013p\u00a0\/opt\/hadoop_scripts<\/li>\n<li>rsync\u00a0\u2013avz\u00a0192.168.1.1::hadoop_s\u00a0\u00a0\u00a0\/opt\/hadoop_scripts<\/li>\n<\/ol>\n<p>2.\u4fee\u6539hostname\uff0c\u6267\u884c\u90e8\u7f72\u811a\u672c CreateDatanode.sh<\/p>\n<ol>\n<li>sh\u00a0\/opt\/hadoop_scripts\/deploy\/AddHostname.sh<\/li>\n<li>sh\u00a0\/opt\/hadoop_scripts\/deploy\/CreateDatanode.sh<\/li>\n<\/ol>\n<p>\u96c6\u7fa4\u521d\u59cb\u5316<\/p>\n<p>\u5230\u8fd9\u91cc\uff0chadoop\u96c6\u7fa4\u7684\u73af\u5883\u5df2\u90e8\u7f72\u5b8c\u6bd5\uff0c\u73b0\u5728\u5f00\u59cb\u521d\u59cb\u5316\u96c6\u7fa4<\/p>\n<p><strong>\u4e00\u3001namenode\u7684HA\u9ad8\u53ef\u7528\u521d\u59cb\u5316<\/strong><\/p>\n<p>1.\u5728namenode\u4e3b\u670d\u52a1\u5668(192.168.1.1)\u6267\u884czookeeper\u7684failover\u529f\u80fd\u683c\u5f0f\u5316<\/p>\n<ol>\n<li>sudo\u00a0\u2013u\u00a0hdfs\u00a0hdfs\u00a0zkfc\u00a0\u2013formatZK<\/li>\n<\/ol>\n<p>2.\u628azookeeper\u96c6\u7fa4\u670d\u52a1\u542f\u52a8(192.168.1.100 \u00a0192.168.1.101 )<\/p>\n<ol>\n<li>\/etc\/init.d\/zookeeper-server\u00a0start<\/li>\n<\/ol>\n<p>3.\u628anamenode\u4e3b\u5907\u670d\u52a1\u5668\u7684zkfc\u670d\u52a1\u8d77\u6765(192.168.1.1 \u00a0192.168.1.2)<\/p>\n<ol>\n<li>\/etc\/init.d\/hadoop-hdfs-zkfc\u00a0start<\/li>\n<\/ol>\n<p>4.\u5728namenode\u4e3b\u670d\u52a1\u5668(192.168.1.1)\u683c\u5f0f\u5316hdfs<\/p>\n<ol>\n<li>#\u786e\u4fdd\u662f\u7528hdfs\u7528\u6237\u683c\u5f0f\u5316<\/li>\n<li>sudo\u00a0-u\u00a0hdfs\u00a0hadoop\u00a0namenode\u00a0\u2013format<\/li>\n<\/ol>\n<p>5.\u7b2c\u4e00\u6b21\u642d\u5efanamenode\u9ad8\u53ef\u7528\uff0c\u9700\u8981\u628aname.dir\u4e0b\u9762\u7684\u6570\u636e\u590d\u5236\u5230namenode\u5907\u670d\u52a1\u5668\uff08\u6b64\u5751\u82b1\u4e86\u597d\u591a\u65f6\u95f4\uff09<\/p>\n<p>a.\u5728\u4e3b\u670d\u52a1\u5668(192.168.1.1)\u6267\u884c<\/p>\n<ol>\n<li>tar\u00a0-zcvPf\u00a0\/tmp\/namedir.tar.gz\u00a0\/data\/hadoop\/dfs\/name\/<\/li>\n<li>nc\u00a0-l\u00a09999\u00a0&lt;\u00a0\/tmp\/namedir.tar.gz<\/li>\n<\/ol>\n<p>b.\u5728\u5907\u670d\u52a1\u5668(192.168.1.2)\u6267\u884c<\/p>\n<ol>\n<li>wget\u00a0192.168.1.1:9999\u00a0-O\u00a0\/tmp\/namedir.tar.gz<\/li>\n<li>tar\u00a0-zxvPf\u00a0\/tmp\/namedir.tar.gz<\/li>\n<\/ol>\n<p>6.\u4e3b\u4ece\u670d\u52a1\u90fd\u542f\u52a8<\/p>\n<ol>\n<li>\/etc\/init.d\/hadoop-hdfs-namenode\u00a0start<\/li>\n<li>\/etc\/init.d\/hadoop-yarn-resourcemanager\u00a0start<\/li>\n<\/ol>\n<p>7.\u67e5\u770bhdfs\u7684web\u754c\u9762<\/p>\n<ol>\n<li>http:\/\/192.168.1.1:9080<\/li>\n<li>http:\/\/192.168.1.2:9080<\/li>\n<li>#\u5982\u679c\u5728web\u754c\u9762\u770b\u5230\u4e24\u4e2anamenode\u90fd\u662fbackup\u72b6\u6001\uff0c\u90a3\u5c31\u662fauto\u00a0fail\u00a0over\u914d\u7f6e\u4e0d\u6210\u529f<\/li>\n<li>#\u67e5\u770bzkfc\u65e5\u5fd7(\/var\/log\/hadoop-hdfs\/hadoop-hdfs-zkfc-nn.dg.s.kingsoft.net.log)<\/li>\n<li>#\u67e5\u770bzookeeper\u96c6\u7fa4\u7684\u65e5\u5fd7(\/var\/log\/zookeeper\/zookeeper.log)<\/li>\n<\/ol>\n<p>8.\u73b0\u5728\u53ef\u4ee5\u5c1d\u8bd5\u5173\u95ednamenode\u4e3b\u670d\u52a1\uff0c\u770b\u662f\u5426\u80fd\u4e3b\u4ece\u5207\u6362<\/p>\n<p><strong>\u4e8c\u3001hdfs\u96c6\u7fa4\u5f00\u542f<\/strong><\/p>\n<p>\u5230\u8fd9\u91cc\uff0c\u6240\u6709hadoop\u90e8\u7f72\u5df2\u5b8c\u6210\uff0c\u73b0\u5728\u5f00\u59cb\u628a\u96c6\u7fa4\u542f\u52a8\uff0c\u9a8c\u8bc1\u6548\u679c<\/p>\n<p>1.\u628a\u6240\u6709datanode\u670d\u52a1\u5668\u542f\u52a8<\/p>\n<ol>\n<li>#\u8fd8\u8bb0\u5f97\u4e4b\u524d\u642d\u5efa\u7684saltstack\u7ba1\u7406\u5de5\u5177\u4e0d\uff0c\u73b0\u5728\u5f00\u59cb\u53d1\u6325\u5b83\u7684\u4f5c\u7528,\u767b\u5f55saltstack\u00a0master(192.168.1.1)\u6267\u884c<\/li>\n<li>salt\u00a0-v\u00a0&#8220;dn*&#8221;\u00a0cmd.run\u00a0&#8220;\/etc\/init.d\/hadoop-hdfs-datanode\u00a0start&#8221;<\/li>\n<\/ol>\n<p>2.\u67e5\u770bhdfs web\u754c\u9762,\u770b\u662f\u5426\u90fd\u6210\u4e3alive nodes<\/p>\n<p>3.\u5982\u679c\u6ca1\u6709\u95ee\u9898\uff0c\u73b0\u5728\u53ef\u4ee5\u5c1d\u8bd5hdfs\u64cd\u4f5c<\/p>\n<ol>\n<li>#\u521b\u5efa\u4e00\u4e2atmp\u76ee\u5f55<\/li>\n<li>sudo\u00a0-u\u00a0hdfs\u00a0hdfs\u00a0dfs\u00a0-mkdir\u00a0\/tmp<\/li>\n<li>#\u521b\u5efa\u4e00\u4e2a10G\u5927\u5c0f\u7684\u7a7a\u6587\u4ef6,\u8ba1\u7b97\u5b83\u7684MD5\u503c\uff0c\u5e76\u653e\u5165hdfs<\/li>\n<li>dd\u00a0if=\/dev\/zero\u00a0of=\/data\/test_10G_file\u00a0bs=1G\u00a0count=10<\/li>\n<li>md5sum\u00a0\/data\/test_10G_file<\/li>\n<li>sudo\u00a0-u\u00a0hdfs\u00a0hdfs\u00a0dfs\u00a0-put\u00a0\/data\/test_10G_file\u00a0\u00a0\/tmp<\/li>\n<li>sudo\u00a0-u\u00a0hdfs\u00a0hdfs\u00a0dfs\u00a0-ls\u00a0\/tmp<\/li>\n<li>#\u73b0\u5728\u53ef\u4ee5\u5c1d\u8bd5\u5173\u95ed\u4e00\u53f0datanode,\u7136\u540e\u628a\u521a\u624d\u7684\u6d4b\u8bd5\u6587\u4ef6\u62c9\u53d6\u51fa\u6765\uff0c\u518d\u7b97\u4e00\u6b21MD5\u770b\u662f\u5426\u4e00\u6837<\/li>\n<li>sudo\u00a0-u\u00a0hdfs\u00a0hdfs\u00a0dfs\u00a0-get\u00a0\/tmp\/test_10G_file\u00a0\/tmp\/<\/li>\n<li>md5sum\u00a0\/tmp\/test_10G_file<\/li>\n<\/ol>\n<p><strong>\u4e09\u3001yarn\u96c6\u7fa4\u5f00\u542f<\/strong><\/p>\n<p>hadoop\u9664\u4e86hdfs\u7528\u4e8e\u5927\u6570\u636e\u7684\u5206\u5e03\u5f0f\u5b58\u50a8\uff0c\u8fd8\u6709\u66f4\u91cd\u8981\u7684\u7ec4\u4ef6\uff0c\u5206\u5e03\u5f0f\u8ba1\u7b97(mapreduce)\u3002\u73b0\u5728\u6211\u4eec\u6765\u628amapreducev2 yarn\u96c6\u7fa4\u542f\u52a8<\/p>\n<p>1.\u5728\u4e3b\u670d\u52a1\u5668\u628aresourcemanager\u670d\u52a1\u8d77\u6765\uff08192.168.1.1\uff09<\/p>\n<ol>\n<li>\/etc\/init.d\/hadoop-yarn-resourcemanager\u00a0start<\/li>\n<\/ol>\n<p>2.\u628a\u6240\u6709nodemanager\u670d\u52a1\u542f\u52a8<\/p>\n<ol>\n<li>#\u8fd8\u662f\u767b\u9646saltstack\u00a0master\uff0c\u6267\u884c<\/li>\n<li>salt\u00a0-v\u00a0&#8220;dn*&#8221;\u00a0cmd.run\u00a0&#8220;\/etc\/init.d\/hadoop-yarn-nodemanager\u00a0start&#8221;<\/li>\n<\/ol>\n<p>3.\u67e5\u770byarn \u4efb\u52a1\u8ffd\u8e2a\u754c\u9762(<a href=\"http:\/\/192.168.1.1:9081\/\" target=\"_blank\">http:\/\/192.168.1.1:9081\/<\/a>),\u770b\u662f\u5426\u6240\u6709nodes\u90fd\u5df2\u52a0\u5165<\/p>\n<p>4.hadoop\u81ea\u5e26\u6709\u57fa\u51c6\u6d4b\u8bd5\u7684mapreduce\u5b9e\u4f8b\uff0c\u6211\u4eec\u5229\u7528\u5b83\u6765\u6d4b\u8bd5yarn\u73af\u5883\u662f\u5426\u6b63\u5e38<\/p>\n<ol>\n<li>#TestDFSIO\u6d4b\u8bd5HDFS\u7684\u8bfb\u5199\u6027\u80fd,\u519910\u4e2a\u6587\u4ef6\uff0c\u6bcf\u4e2a\u6587\u4ef61G.<\/li>\n<li>su\u00a0hdfs\u00a0&#8211;<\/li>\n<li>hadoop\u00a0jar\u00a0\/usr\/lib\/hadoop-mapreduce\/hadoop-mapreduce-client-jobclient-2.0.0-cdh4.2.1-tests.jar\u00a0TestDFSIO\u00a0\u00a0-write\u00a0-nrFiles\u00a010\u00a0-fileSize\u00a01000<\/li>\n<li>#Sort\u6d4b\u8bd5MapReduce<\/li>\n<li>##\u5411random-data\u76ee\u5f55\u8f93\u51fa\u6570\u636e<\/li>\n<li>hadoop\u00a0jar\u00a0\/usr\/lib\/hadoop-mapreduce\/hadoop-mapreduce-examples.jar\u00a0randomwriter\u00a0\u00a0random-data<\/li>\n<li>##\u8fd0\u884csort\u7a0b\u5e8f<\/li>\n<li>hadoop\u00a0jar\u00a0\/usr\/lib\/hadoop-mapreduce\/hadoop-mapreduce-examples.jar\u00a0sort\u00a0random-data\u00a0sorted-data<\/li>\n<li>##\u9a8c\u8bc1sorted-data\u00a0\u6587\u4ef6\u662f\u5426\u6392\u597d\u5e8f<\/li>\n<li>hadoop\u00a0jar\u00a0\/usr\/lib\/hadoop-mapreduce\/hadoop-mapreduce-client-jobclient-2.0.0-cdh4.2.1-tests.jar\u00a0testmapredsort\u00a0-sortInput\u00a0random-data\u00a0\\<\/li>\n<li>-sortOutput\u00a0sorted-data<\/li>\n<\/ol>\n<p><strong>\u603b\u7ed3<\/strong><\/p>\n<p>\u5728\u6298\u817ehadoop\u96c6\u7fa4\u7684\u90e8\u7f72\u662f\u8fd8\u662f\u9047\u5230\u4e86\u5f88\u591a\u5751\uff0c\u6253\u7b97\u4e0b\u7bc7\u5199\u81ea\u5df1\u6240\u906d\u9047\u7684\u95ee\u9898\u3002\u901a\u8fc7\u672c\u6587\u90e8\u7f72\u9047\u5230\u95ee\u9898\u7684\u53ef\u4ee5\u8054\u7cfb\u4e00\u4e0b\u6211,\u4e92\u76f8\u4ea4\u6d41\u4e00\u4e0b\u3002QQ:83766787\u3002\u5f53\u7136\u4e5f\u6b22\u8fce\u5927\u5bb6\u4e00\u8d77\u4fee\u6539\u90e8\u7f72\u7684\u811a\u672c\uff0cgit\u5730\u5740\u662f\uff1a<a href=\"http:\/\/git.oschina.net\/snake1361222\/hadoop_scripts\" target=\"_blank\">http:\/\/git.oschina.net\/snake1361222\/hadoop_scripts<\/a><\/p>\n<p>\u539f\u6587\u94fe\u63a5\uff1a<a href=\"http:\/\/lxcong.blog.51cto.com\/7485244\/1241004\" target=\"_blank\">http:\/\/lxcong.blog.51cto.com\/7485244\/1241004<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u524d\u8a00 \u6298\u817e\u4e86\u4e00\u6bb5\u65f6\u95f4hadoop\u7684\u90e8\u7f72\u7ba1\u7406\uff0c\u5199\u4e0b\u6b64\u7cfb\u5217\u535a\u5ba2\u8bb0\u5f55\u4e00\u4e0b\u3002 \u4e3a\u4e86\u907f\u514d\u5404\u4f4d\u505a\u90e8\u7f72\u8fd9\u79cd\u91cd\u590d\u6027\u7684\u52b3\u52a8\uff0c\u6211\u5df2 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[230,124],"tags":[231,128],"_links":{"self":[{"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/356"}],"collection":[{"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=356"}],"version-history":[{"count":0,"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/356\/revisions"}],"wp:attachment":[{"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=356"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=356"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=356"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}