{"id":1493,"date":"2017-11-03T16:11:18","date_gmt":"2017-11-03T08:11:18","guid":{"rendered":"https:\/\/yanjingang.com\/blog\/?p=1493"},"modified":"2019-08-15T20:26:36","modified_gmt":"2019-08-15T12:26:36","slug":"kafka%e6%b7%b1%e5%ba%a6%e8%a7%a3%e6%9e%90","status":"publish","type":"post","link":"https:\/\/yanjingang.com\/blog\/?p=1493","title":{"rendered":"Kafka\u6df1\u5ea6\u89e3\u6790"},"content":{"rendered":"<h2>\u80cc\u666f\u4ecb\u7ecd<\/h2>\n<h2>Kafka\u7b80\u4ecb<\/h2>\n<p>Kafka\u662f\u4e00\u79cd\u5206\u5e03\u5f0f\u7684\uff0c\u57fa\u4e8e\u53d1\u5e03\/\u8ba2\u9605\u7684\u6d88\u606f\u7cfb\u7edf\u3002\u4e3b\u8981\u8bbe\u8ba1\u76ee\u6807\u5982\u4e0b\uff1a<\/p>\n<ul>\n<li>\u4ee5\u65f6\u95f4\u590d\u6742\u5ea6\u4e3aO(1)\u7684\u65b9\u5f0f\u63d0\u4f9b\u6d88\u606f\u6301\u4e45\u5316\u80fd\u529b\uff0c\u5e76\u4fdd\u8bc1\u5373\u4f7f\u5bf9TB\u7ea7\u4ee5\u4e0a\u6570\u636e\u4e5f\u80fd\u4fdd\u8bc1\u5e38\u6570\u65f6\u95f4\u7684\u8bbf\u95ee\u6027\u80fd<\/li>\n<li>\u9ad8\u541e\u5410\u7387\u3002\u5373\u4f7f\u5728\u975e\u5e38\u5ec9\u4ef7\u7684\u5546\u7528\u673a\u5668\u4e0a\u4e5f\u80fd\u505a\u5230\u5355\u673a\u652f\u6301\u6bcf\u79d2100K\u6761\u6d88\u606f\u7684\u4f20\u8f93<\/li>\n<li>\u652f\u6301Kafka Server\u95f4\u7684\u6d88\u606f\u5206\u533a\uff0c\u53ca\u5206\u5e03\u5f0f\u6d88\u606f\u6d88\u8d39\uff0c\u540c\u65f6\u4fdd\u8bc1\u6bcf\u4e2apartition\u5185\u7684\u6d88\u606f\u987a\u5e8f\u4f20\u8f93<\/li>\n<li>\u540c\u65f6\u652f\u6301\u79bb\u7ebf\u6570\u636e\u5904\u7406\u548c\u5b9e\u65f6\u6570\u636e\u5904\u7406<\/li>\n<\/ul>\n<h2>\u4e3a\u4ec0\u4e48\u8981\u7528Message Queue<\/h2>\n<ul>\n<li><b>\u89e3\u8026<\/b><br \/>\n\u5728\u9879\u76ee\u542f\u52a8\u4e4b\u521d\u6765\u9884\u6d4b\u5c06\u6765\u9879\u76ee\u4f1a\u78b0\u5230\u4ec0\u4e48\u9700\u6c42\uff0c\u662f\u6781\u5176\u56f0\u96be\u7684\u3002\u6d88\u606f\u961f\u5217\u5728\u5904\u7406\u8fc7\u7a0b\u4e2d\u95f4\u63d2\u5165\u4e86\u4e00\u4e2a\u9690\u542b\u7684\u3001\u57fa\u4e8e\u6570\u636e\u7684\u63a5\u53e3\u5c42\uff0c\u4e24\u8fb9\u7684\u5904\u7406\u8fc7\u7a0b\u90fd\u8981\u5b9e\u73b0\u8fd9\u4e00\u63a5\u53e3\u3002\u8fd9\u5141\u8bb8\u4f60\u72ec\u7acb\u7684\u6269\u5c55\u6216\u4fee\u6539\u4e24\u8fb9\u7684\u5904\u7406\u8fc7\u7a0b\uff0c\u53ea\u8981\u786e\u4fdd\u5b83\u4eec\u9075\u5b88\u540c\u6837\u7684\u63a5\u53e3\u7ea6\u675f<\/li>\n<li><strong>\u5197\u4f59<\/strong><br \/>\n\u6709\u65f6\u5728\u5904\u7406\u6570\u636e\u7684\u65f6\u5019\u5904\u7406\u8fc7\u7a0b\u4f1a\u5931\u8d25\u3002\u9664\u975e\u6570\u636e\u88ab\u6301\u4e45\u5316\uff0c\u5426\u5219\u5c06\u6c38\u8fdc\u4e22\u5931\u3002\u6d88\u606f\u961f\u5217\u628a\u6570\u636e\u8fdb\u884c\u6301\u4e45\u5316\u76f4\u5230\u5b83\u4eec\u5df2\u7ecf\u88ab\u5b8c\u5168\u5904\u7406\uff0c\u901a\u8fc7\u8fd9\u4e00\u65b9\u5f0f\u89c4\u907f\u4e86\u6570\u636e\u4e22\u5931\u98ce\u9669\u3002\u5728\u88ab\u8bb8\u591a\u6d88\u606f\u961f\u5217\u6240\u91c7\u7528\u7684\u201d\u63d2\u5165-\u83b7\u53d6-\u5220\u9664\u201d\u8303\u5f0f\u4e2d\uff0c\u5728\u628a\u4e00\u4e2a\u6d88\u606f\u4ece\u961f\u5217\u4e2d\u5220\u9664\u4e4b\u524d\uff0c\u9700\u8981\u4f60\u7684\u5904\u7406\u8fc7\u7a0b\u660e\u786e\u7684\u6307\u51fa\u8be5\u6d88\u606f\u5df2\u7ecf\u88ab\u5904\u7406\u5b8c\u6bd5\uff0c\u786e\u4fdd\u4f60\u7684\u6570\u636e\u88ab\u5b89\u5168\u7684\u4fdd\u5b58\u76f4\u5230\u4f60\u4f7f\u7528\u5b8c\u6bd5\u3002<\/li>\n<li><strong>\u6269\u5c55\u6027<\/strong><br \/>\n\u56e0\u4e3a\u6d88\u606f\u961f\u5217\u89e3\u8026\u4e86\u4f60\u7684\u5904\u7406\u8fc7\u7a0b\uff0c\u6240\u4ee5\u589e\u5927\u6d88\u606f\u5165\u961f\u548c\u5904\u7406\u7684\u9891\u7387\u662f\u5f88\u5bb9\u6613\u7684\uff1b\u53ea\u8981\u53e6\u5916\u589e\u52a0\u5904\u7406\u8fc7\u7a0b\u5373\u53ef\u3002\u4e0d\u9700\u8981\u6539\u53d8\u4ee3\u7801\u3001\u4e0d\u9700\u8981\u8c03\u8282\u53c2\u6570\u3002\u6269\u5c55\u5c31\u50cf\u8c03\u5927\u7535\u529b\u6309\u94ae\u4e00\u6837\u7b80\u5355\u3002<\/li>\n<li><strong>\u7075\u6d3b\u6027 &amp; \u5cf0\u503c\u5904\u7406\u80fd\u529b<\/strong><br \/>\n\u5728\u8bbf\u95ee\u91cf\u5267\u589e\u7684\u60c5\u51b5\u4e0b\uff0c\u5e94\u7528\u4ecd\u7136\u9700\u8981\u7ee7\u7eed\u53d1\u6325\u4f5c\u7528\uff0c\u4f46\u662f\u8fd9\u6837\u7684\u7a81\u53d1\u6d41\u91cf\u5e76\u4e0d\u5e38\u89c1\uff1b\u5982\u679c\u4e3a\u4ee5\u80fd\u5904\u7406\u8fd9\u7c7b\u5cf0\u503c\u8bbf\u95ee\u4e3a\u6807\u51c6\u6765\u6295\u5165\u8d44\u6e90\u968f\u65f6\u5f85\u547d\u65e0\u7591\u662f\u5de8\u5927\u7684\u6d6a\u8d39\u3002\u4f7f\u7528\u6d88\u606f\u961f\u5217\u80fd\u591f\u4f7f\u5173\u952e\u7ec4\u4ef6\u9876\u4f4f\u589e\u957f\u7684\u8bbf\u95ee\u538b\u529b\uff0c\u800c\u4e0d\u662f\u56e0\u4e3a\u8d85\u51fa\u8d1f\u8377\u7684\u8bf7\u6c42\u800c\u5b8c\u5168\u5d29\u6e83\u3002<\/li>\n<li><strong>\u53ef\u6062\u590d\u6027<\/strong><br \/>\n\u5f53\u4f53\u7cfb\u7684\u4e00\u90e8\u5206\u7ec4\u4ef6\u5931\u6548\uff0c\u4e0d\u4f1a\u5f71\u54cd\u5230\u6574\u4e2a\u7cfb\u7edf\u3002\u6d88\u606f\u961f\u5217\u964d\u4f4e\u4e86\u8fdb\u7a0b\u95f4\u7684\u8026\u5408\u5ea6\uff0c\u6240\u4ee5\u5373\u4f7f\u4e00\u4e2a\u5904\u7406\u6d88\u606f\u7684\u8fdb\u7a0b\u6302\u6389\uff0c\u52a0\u5165\u961f\u5217\u4e2d\u7684\u6d88\u606f\u4ecd\u7136\u53ef\u4ee5\u5728\u7cfb\u7edf\u6062\u590d\u540e\u88ab\u5904\u7406\u3002\u800c\u8fd9\u79cd\u5141\u8bb8\u91cd\u8bd5\u6216\u8005\u5ef6\u540e\u5904\u7406\u8bf7\u6c42\u7684\u80fd\u529b\u901a\u5e38\u662f\u9020\u5c31\u4e00\u4e2a\u7565\u611f\u4e0d\u4fbf\u7684\u7528\u6237\u548c\u4e00\u4e2a\u6cae\u4e27\u900f\u9876\u7684\u7528\u6237\u4e4b\u95f4\u7684\u533a\u522b\u3002<\/li>\n<li><strong>\u9001\u8fbe\u4fdd\u8bc1<\/strong><br \/>\n\u6d88\u606f\u961f\u5217\u63d0\u4f9b\u7684\u5197\u4f59\u673a\u5236\u4fdd\u8bc1\u4e86\u6d88\u606f\u80fd\u88ab\u5b9e\u9645\u7684\u5904\u7406\uff0c\u53ea\u8981\u4e00\u4e2a\u8fdb\u7a0b\u8bfb\u53d6\u4e86\u8be5\u961f\u5217\u5373\u53ef\u3002\u5728\u6b64\u57fa\u7840\u4e0a\uff0cIronMQ\u63d0\u4f9b\u4e86\u4e00\u4e2a\u201d\u53ea\u9001\u8fbe\u4e00\u6b21\u201d\u4fdd\u8bc1\u3002\u65e0\u8bba\u6709\u591a\u5c11\u8fdb\u7a0b\u5728\u4ece\u961f\u5217\u4e2d\u9886\u53d6\u6570\u636e\uff0c\u6bcf\u4e00\u4e2a\u6d88\u606f\u53ea\u80fd\u88ab\u5904\u7406\u4e00\u6b21\u3002\u8fd9\u4e4b\u6240\u4ee5\u6210\u4e3a\u53ef\u80fd\uff0c\u662f\u56e0\u4e3a\u83b7\u53d6\u4e00\u4e2a\u6d88\u606f\u53ea\u662f\u201d\u9884\u5b9a\u201d\u4e86\u8fd9\u4e2a\u6d88\u606f\uff0c\u6682\u65f6\u628a\u5b83\u79fb\u51fa\u4e86\u961f\u5217\u3002\u9664\u975e\u5ba2\u6237\u7aef\u660e\u786e\u7684\u8868\u793a\u5df2\u7ecf\u5904\u7406\u5b8c\u4e86\u8fd9\u4e2a\u6d88\u606f\uff0c\u5426\u5219\u8fd9\u4e2a\u6d88\u606f\u4f1a\u88ab\u653e\u56de\u961f\u5217\u4e2d\u53bb\uff0c\u5728\u4e00\u6bb5\u53ef\u914d\u7f6e\u7684\u65f6\u95f4\u4e4b\u540e\u53ef\u518d\u6b21\u88ab\u5904\u7406\u3002<\/li>\n<\/ul>\n<ul>\n<li><strong>\u987a\u5e8f\u4fdd\u8bc1<\/strong><br \/>\n\u5728\u8bb8\u591a\u60c5\u51b5\u4e0b\uff0c\u6570\u636e\u5904\u7406\u7684\u987a\u5e8f\u90fd\u5f88\u91cd\u8981\u3002\u6d88\u606f\u961f\u5217\u672c\u6765\u5c31\u662f\u6392\u5e8f\u7684\uff0c\u5e76\u4e14\u80fd\u4fdd\u8bc1\u6570\u636e\u4f1a\u6309\u7167\u7279\u5b9a\u7684\u987a\u5e8f\u6765\u5904\u7406\u3002IronMO\u4fdd\u8bc1\u6d88\u606f\u6d46\u7cca\u901a\u8fc7FIFO\uff08\u5148\u8fdb\u5148\u51fa\uff09\u7684\u987a\u5e8f\u6765\u5904\u7406\uff0c\u56e0\u6b64\u6d88\u606f\u5728\u961f\u5217\u4e2d\u7684\u4f4d\u7f6e\u5c31\u662f\u4ece\u961f\u5217\u4e2d\u68c0\u7d22\u4ed6\u4eec\u7684\u4f4d\u7f6e\u3002<\/li>\n<li><strong>\u7f13\u51b2<\/strong><br \/>\n\u5728\u4efb\u4f55\u91cd\u8981\u7684\u7cfb\u7edf\u4e2d\uff0c\u90fd\u4f1a\u6709\u9700\u8981\u4e0d\u540c\u7684\u5904\u7406\u65f6\u95f4\u7684\u5143\u7d20\u3002\u4f8b\u5982,\u52a0\u8f7d\u4e00\u5f20\u56fe\u7247\u6bd4\u5e94\u7528\u8fc7\u6ee4\u5668\u82b1\u8d39\u66f4\u5c11\u7684\u65f6\u95f4\u3002\u6d88\u606f\u961f\u5217\u901a\u8fc7\u4e00\u4e2a\u7f13\u51b2\u5c42\u6765\u5e2e\u52a9\u4efb\u52a1\u6700\u9ad8\u6548\u7387\u7684\u6267\u884c\u2014\u5199\u5165\u961f\u5217\u7684\u5904\u7406\u4f1a\u5c3d\u53ef\u80fd\u7684\u5feb\u901f\uff0c\u800c\u4e0d\u53d7\u4ece\u961f\u5217\u8bfb\u7684\u9884\u5907\u5904\u7406\u7684\u7ea6\u675f\u3002\u8be5\u7f13\u51b2\u6709\u52a9\u4e8e\u63a7\u5236\u548c\u4f18\u5316\u6570\u636e\u6d41\u7ecf\u8fc7\u7cfb\u7edf\u7684\u901f\u5ea6\u3002<\/li>\n<li><strong>\u7406\u89e3\u6570\u636e\u6d41<\/strong><br \/>\n\u5728\u4e00\u4e2a\u5206\u5e03\u5f0f\u7cfb\u7edf\u91cc\uff0c\u8981\u5f97\u5230\u4e00\u4e2a\u5173\u4e8e\u7528\u6237\u64cd\u4f5c\u4f1a\u7528\u591a\u957f\u65f6\u95f4\u53ca\u5176\u539f\u56e0\u7684\u603b\u4f53\u5370\u8c61\uff0c\u662f\u4e2a\u5de8\u5927\u7684\u6311\u6218\u3002\u6d88\u606f\u7cfb\u5217\u901a\u8fc7\u6d88\u606f\u88ab\u5904\u7406\u7684\u9891\u7387\uff0c\u6765\u65b9\u4fbf\u7684\u8f85\u52a9\u786e\u5b9a\u90a3\u4e9b\u8868\u73b0\u4e0d\u4f73\u7684\u5904\u7406\u8fc7\u7a0b\u6216\u9886\u57df\uff0c\u8fd9\u4e9b\u5730\u65b9\u7684\u6570\u636e\u6d41\u90fd\u4e0d\u591f\u4f18\u5316\u3002<\/li>\n<li><strong>\u5f02\u6b65\u901a\u4fe1<\/strong><br \/>\n\u5f88\u591a\u65f6\u5019\uff0c\u4f60\u4e0d\u60f3\u4e5f\u4e0d\u9700\u8981\u7acb\u5373\u5904\u7406\u6d88\u606f\u3002\u6d88\u606f\u961f\u5217\u63d0\u4f9b\u4e86\u5f02\u6b65\u5904\u7406\u673a\u5236\uff0c\u5141\u8bb8\u4f60\u628a\u4e00\u4e2a\u6d88\u606f\u653e\u5165\u961f\u5217\uff0c\u4f46\u5e76\u4e0d\u7acb\u5373\u5904\u7406\u5b83\u3002\u4f60\u60f3\u5411\u961f\u5217\u4e2d\u653e\u5165\u591a\u5c11\u6d88\u606f\u5c31\u653e\u591a\u5c11\uff0c\u7136\u540e\u5728\u4f60\u4e50\u610f\u7684\u65f6\u5019\u518d\u53bb\u5904\u7406\u5b83\u4eec\u3002<\/li>\n<\/ul>\n<h2>\u5e38\u7528Message Queue\u5bf9\u6bd4<\/h2>\n<ul>\n<li><strong>RabbitMQ<\/strong><br \/>\nRabbitMQ\u662f\u4f7f\u7528Erlang\u7f16\u5199\u7684\u4e00\u4e2a\u5f00\u6e90\u7684\u6d88\u606f\u961f\u5217\uff0c\u672c\u8eab\u652f\u6301\u5f88\u591a\u7684\u534f\u8bae\uff1aAMQP\uff0cXMPP, SMTP, STOMP\uff0c\u4e5f\u6b63\u56e0\u5982\u6b64\uff0c\u5b83\u975e\u5e38\u91cd\u91cf\u7ea7\uff0c\u66f4\u9002\u5408\u4e8e\u4f01\u4e1a\u7ea7\u7684\u5f00\u53d1\u3002\u540c\u65f6\u5b9e\u73b0\u4e86Broker\u6784\u67b6\uff0c\u8fd9\u610f\u5473\u7740\u6d88\u606f\u5728\u53d1\u9001\u7ed9\u5ba2\u6237\u7aef\u65f6\u5148\u5728\u4e2d\u5fc3\u961f\u5217\u6392\u961f\u3002\u5bf9\u8def\u7531\uff0c\u8d1f\u8f7d\u5747\u8861\u6216\u8005\u6570\u636e\u6301\u4e45\u5316\u90fd\u6709\u5f88\u597d\u7684\u652f\u6301\u3002<\/li>\n<li><strong>Redis<\/strong><br \/>\nRedis\u662f\u4e00\u4e2a\u57fa\u4e8eKey-Value\u5bf9\u7684NoSQL\u6570\u636e\u5e93\uff0c\u5f00\u53d1\u7ef4\u62a4\u5f88\u6d3b\u8dc3\u3002\u867d\u7136\u5b83\u662f\u4e00\u4e2aKey-Value\u6570\u636e\u5e93\u5b58\u50a8\u7cfb\u7edf\uff0c\u4f46\u5b83\u672c\u8eab\u652f\u6301MQ\u529f\u80fd\uff0c\u6240\u4ee5\u5b8c\u5168\u53ef\u4ee5\u5f53\u505a\u4e00\u4e2a\u8f7b\u91cf\u7ea7\u7684\u961f\u5217\u670d\u52a1\u6765\u4f7f\u7528\u3002\u5bf9\u4e8eRabbitMQ\u548cRedis\u7684\u5165\u961f\u548c\u51fa\u961f\u64cd\u4f5c\uff0c\u5404\u6267\u884c100\u4e07\u6b21\uff0c\u6bcf10\u4e07\u6b21\u8bb0\u5f55\u4e00\u6b21\u6267\u884c\u65f6\u95f4\u3002\u6d4b\u8bd5\u6570\u636e\u5206\u4e3a128Bytes\u3001512Bytes\u30011K\u548c10K\u56db\u4e2a\u4e0d\u540c\u5927\u5c0f\u7684\u6570\u636e\u3002\u5b9e\u9a8c\u8868\u660e\uff1a\u5165\u961f\u65f6\uff0c\u5f53\u6570\u636e\u6bd4\u8f83\u5c0f\u65f6Redis\u7684\u6027\u80fd\u8981\u9ad8\u4e8eRabbitMQ\uff0c\u800c\u5982\u679c\u6570\u636e\u5927\u5c0f\u8d85\u8fc7\u4e8610K\uff0cRedis\u5219\u6162\u7684\u65e0\u6cd5\u5fcd\u53d7\uff1b\u51fa\u961f\u65f6\uff0c\u65e0\u8bba\u6570\u636e\u5927\u5c0f\uff0cRedis\u90fd\u8868\u73b0\u51fa\u975e\u5e38\u597d\u7684\u6027\u80fd\uff0c\u800cRabbitMQ\u7684\u51fa\u961f\u6027\u80fd\u5219\u8fdc\u4f4e\u4e8eRedis\u3002<\/li>\n<li><strong>ZeroMQ<\/strong><br \/>\nZeroMQ\u53f7\u79f0\u6700\u5feb\u7684\u6d88\u606f\u961f\u5217\u7cfb\u7edf\uff0c\u5c24\u5176\u9488\u5bf9\u5927\u541e\u5410\u91cf\u7684\u9700\u6c42\u573a\u666f\u3002ZMQ\u80fd\u591f\u5b9e\u73b0RabbitMQ\u4e0d\u64c5\u957f\u7684\u9ad8\u7ea7\/\u590d\u6742\u7684\u961f\u5217\uff0c\u4f46\u662f\u5f00\u53d1\u4eba\u5458\u9700\u8981\u81ea\u5df1\u7ec4\u5408\u591a\u79cd\u6280\u672f\u6846\u67b6\uff0c\u6280\u672f\u4e0a\u7684\u590d\u6742\u5ea6\u662f\u5bf9\u8fd9MQ\u80fd\u591f\u5e94\u7528\u6210\u529f\u7684\u6311\u6218\u3002ZeroMQ\u5177\u6709\u4e00\u4e2a\u72ec\u7279\u7684\u975e\u4e2d\u95f4\u4ef6\u7684\u6a21\u5f0f\uff0c\u4f60\u4e0d\u9700\u8981\u5b89\u88c5\u548c\u8fd0\u884c\u4e00\u4e2a\u6d88\u606f\u670d\u52a1\u5668\u6216\u4e2d\u95f4\u4ef6\uff0c\u56e0\u4e3a\u4f60\u7684\u5e94\u7528\u7a0b\u5e8f\u5c06\u626e\u6f14\u4e86\u8fd9\u4e2a\u670d\u52a1\u89d2\u8272\u3002\u4f60\u53ea\u9700\u8981\u7b80\u5355\u7684\u5f15\u7528ZeroMQ\u7a0b\u5e8f\u5e93\uff0c\u53ef\u4ee5\u4f7f\u7528NuGet\u5b89\u88c5\uff0c\u7136\u540e\u4f60\u5c31\u53ef\u4ee5\u6109\u5feb\u7684\u5728\u5e94\u7528\u7a0b\u5e8f\u4e4b\u95f4\u53d1\u9001\u6d88\u606f\u4e86\u3002\u4f46\u662fZeroMQ\u4ec5\u63d0\u4f9b\u975e\u6301\u4e45\u6027\u7684\u961f\u5217\uff0c\u4e5f\u5c31\u662f\u8bf4\u5982\u679cdown\u673a\uff0c\u6570\u636e\u5c06\u4f1a\u4e22\u5931\u3002\u5176\u4e2d\uff0cTwitter\u7684Storm\u4e2d\u9ed8\u8ba4\u4f7f\u7528ZeroMQ\u4f5c\u4e3a\u6570\u636e\u6d41\u7684\u4f20\u8f93\u3002<\/li>\n<li><strong>ActiveMQ<\/strong><br \/>\nActiveMQ\u662fApache\u4e0b\u7684\u4e00\u4e2a\u5b50\u9879\u76ee\u3002 \u7c7b\u4f3c\u4e8eZeroMQ\uff0c\u5b83\u80fd\u591f\u4ee5\u4ee3\u7406\u4eba\u548c\u70b9\u5bf9\u70b9\u7684\u6280\u672f\u5b9e\u73b0\u961f\u5217\u3002\u540c\u65f6\u7c7b\u4f3c\u4e8eRabbitMQ\uff0c\u5b83\u5c11\u91cf\u4ee3\u7801\u5c31\u53ef\u4ee5\u9ad8\u6548\u5730\u5b9e\u73b0\u9ad8\u7ea7\u5e94\u7528\u573a\u666f\u3002<\/li>\n<li><strong>Kafka\/Jafka<\/strong><br \/>\nKafka\u662fApache\u4e0b\u7684\u4e00\u4e2a\u5b50\u9879\u76ee\uff0c\u662f\u4e00\u4e2a\u9ad8\u6027\u80fd\u8de8\u8bed\u8a00\u5206\u5e03\u5f0fPublish\/Subscribe\u6d88\u606f\u961f\u5217\u7cfb\u7edf\uff0c\u800cJafka\u662f\u5728Kafka\u4e4b\u4e0a\u5b75\u5316\u800c\u6765\u7684\uff0c\u5373Kafka\u7684\u4e00\u4e2a\u5347\u7ea7\u7248\u3002\u5177\u6709\u4ee5\u4e0b\u7279\u6027\uff1a\u5feb\u901f\u6301\u4e45\u5316\uff0c\u53ef\u4ee5\u5728O(1)\u7684\u7cfb\u7edf\u5f00\u9500\u4e0b\u8fdb\u884c\u6d88\u606f\u6301\u4e45\u5316\uff1b\u9ad8\u541e\u5410\uff0c\u5728\u4e00\u53f0\u666e\u901a\u7684\u670d\u52a1\u5668\u4e0a\u65e2\u53ef\u4ee5\u8fbe\u523010W\/s\u7684\u541e\u5410\u901f\u7387\uff1b\u5b8c\u5168\u7684\u5206\u5e03\u5f0f\u7cfb\u7edf\uff0cBroker\u3001Producer\u3001Consumer\u90fd\u539f\u751f\u81ea\u52a8\u652f\u6301\u5206\u5e03\u5f0f\uff0c\u81ea\u52a8\u5b9e\u73b0\u590d\u6742\u5747\u8861\uff1b\u652f\u6301Hadoop\u6570\u636e\u5e76\u884c\u52a0\u8f7d\uff0c\u5bf9\u4e8e\u50cfHadoop\u7684\u4e00\u6837\u7684\u65e5\u5fd7\u6570\u636e\u548c\u79bb\u7ebf\u5206\u6790\u7cfb\u7edf\uff0c\u4f46\u53c8\u8981\u6c42\u5b9e\u65f6\u5904\u7406\u7684\u9650\u5236\uff0c\u8fd9\u662f\u4e00\u4e2a\u53ef\u884c\u7684\u89e3\u51b3\u65b9\u6848\u3002Kafka\u901a\u8fc7Hadoop\u7684\u5e76\u884c\u52a0\u8f7d\u673a\u5236\u6765\u7edf\u4e00\u4e86\u5728\u7ebf\u548c\u79bb\u7ebf\u7684\u6d88\u606f\u5904\u7406\uff0c\u8fd9\u4e00\u70b9\u4e5f\u662f\u672c\u8bfe\u9898\u6240\u7814\u7a76\u7cfb\u7edf\u6240\u770b\u91cd\u7684\u3002Apache Kafka\u76f8\u5bf9\u4e8eActiveMQ\u662f\u4e00\u4e2a\u975e\u5e38\u8f7b\u91cf\u7ea7\u7684\u6d88\u606f\u7cfb\u7edf\uff0c\u9664\u4e86\u6027\u80fd\u975e\u5e38\u597d\u4e4b\u5916\uff0c\u8fd8\u662f\u4e00\u4e2a\u5de5\u4f5c\u826f\u597d\u7684\u5206\u5e03\u5f0f\u7cfb\u7edf\u3002<\/li>\n<\/ul>\n<h2>Kafka\u89e3\u6790<\/h2>\n<h2>Terminology<\/h2>\n<ul>\n<li><strong>Broker<\/strong><br \/>\nKafka\u96c6\u7fa4\u5305\u542b\u4e00\u4e2a\u6216\u591a\u4e2a\u670d\u52a1\u5668\uff0c\u8fd9\u79cd\u670d\u52a1\u5668\u88ab\u79f0\u4e3abroker<\/li>\n<li><strong>Topic<\/strong><br \/>\n\u6bcf\u6761\u53d1\u5e03\u5230Kafka\u96c6\u7fa4\u7684\u6d88\u606f\u90fd\u6709\u4e00\u4e2a\u7c7b\u522b\uff0c\u8fd9\u4e2a\u7c7b\u522b\u88ab\u79f0\u4e3atopic\u3002\uff08\u7269\u7406\u4e0a\u4e0d\u540ctopic\u7684\u6d88\u606f\u5206\u5f00\u5b58\u50a8\uff0c\u903b\u8f91\u4e0a\u4e00\u4e2atopic\u7684\u6d88\u606f\u867d\u7136\u4fdd\u5b58\u4e8e\u4e00\u4e2a\u6216\u591a\u4e2abroker\u4e0a\u4f46\u7528\u6237\u53ea\u9700\u6307\u5b9a\u6d88\u606f\u7684topic\u5373\u53ef\u751f\u4ea7\u6216\u6d88\u8d39\u6570\u636e\u800c\u4e0d\u5fc5\u5173\u5fc3\u6570\u636e\u5b58\u4e8e\u4f55\u5904\uff09<\/li>\n<li><strong>Partition<\/strong><br \/>\nparition\u662f\u7269\u7406\u4e0a\u7684\u6982\u5ff5\uff0c\u6bcf\u4e2atopic\u5305\u542b\u4e00\u4e2a\u6216\u591a\u4e2apartition\uff0c\u521b\u5efatopic\u65f6\u53ef\u6307\u5b9aparition\u6570\u91cf\u3002\u6bcf\u4e2apartition\u5bf9\u5e94\u4e8e\u4e00\u4e2a\u6587\u4ef6\u5939\uff0c\u8be5\u6587\u4ef6\u5939\u4e0b\u5b58\u50a8\u8be5partition\u7684\u6570\u636e\u548c\u7d22\u5f15\u6587\u4ef6<\/li>\n<li><strong>Producer<\/strong><br \/>\n\u8d1f\u8d23\u53d1\u5e03\u6d88\u606f\u5230Kafka broker<\/li>\n<li><strong>Consumer<\/strong><br \/>\n\u6d88\u8d39\u6d88\u606f\u3002\u6bcf\u4e2aconsumer\u5c5e\u4e8e\u4e00\u4e2a\u7279\u5b9a\u7684consuer group\uff08\u53ef\u4e3a\u6bcf\u4e2aconsumer\u6307\u5b9agroup name\uff0c\u82e5\u4e0d\u6307\u5b9agroup name\u5219\u5c5e\u4e8e\u9ed8\u8ba4\u7684group\uff09\u3002\u4f7f\u7528consumer high level API\u65f6\uff0c\u540c\u4e00topic\u7684\u4e00\u6761\u6d88\u606f\u53ea\u80fd\u88ab\u540c\u4e00\u4e2aconsumer group\u5185\u7684\u4e00\u4e2aconsumer\u6d88\u8d39\uff0c\u4f46\u591a\u4e2aconsumer group\u53ef\u540c\u65f6\u6d88\u8d39\u8fd9\u4e00\u6d88\u606f\u3002<\/li>\n<\/ul>\n<h2>Kafka\u67b6\u6784<\/h2>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1494\" rel=\"attachment wp-att-1494\"><img loading=\"lazy\" class=\"alignnone size-full wp-image-1494\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/1-1.png\" alt=\"\" width=\"781\" height=\"368\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/1-1.png 781w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/1-1-300x141.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/1-1-768x362.png 768w\" sizes=\"(max-width: 781px) 100vw, 781px\" \/><\/a>\u5982\u4e0a\u56fe\u6240\u793a\uff0c\u4e00\u4e2a\u5178\u578b\u7684kafka\u96c6\u7fa4\u4e2d\u5305\u542b\u82e5\u5e72producer\uff08\u53ef\u4ee5\u662fweb\u524d\u7aef\u4ea7\u751f\u7684page view\uff0c\u6216\u8005\u662f\u670d\u52a1\u5668\u65e5\u5fd7\uff0c\u7cfb\u7edfCPU\u3001memory\u7b49\uff09\uff0c\u82e5\u5e72broker\uff08Kafka\u652f\u6301\u6c34\u5e73\u6269\u5c55\uff0c\u4e00\u822cbroker\u6570\u91cf\u8d8a\u591a\uff0c\u96c6\u7fa4\u541e\u5410\u7387\u8d8a\u9ad8\uff09\uff0c\u82e5\u5e72consumer group\uff0c\u4ee5\u53ca\u4e00\u4e2a\u00a0<a href=\"http:\/\/zookeeper.apache.org\/\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">Zookeeper<\/a>\u00a0\u96c6\u7fa4\u3002Kafka\u901a\u8fc7Zookeeper\u7ba1\u7406\u96c6\u7fa4\u914d\u7f6e\uff0c\u9009\u4e3eleader\uff0c\u4ee5\u53ca\u5728consumer group\u53d1\u751f\u53d8\u5316\u65f6\u8fdb\u884crebalance\u3002producer\u4f7f\u7528push\u6a21\u5f0f\u5c06\u6d88\u606f\u53d1\u5e03\u5230broker\uff0cconsumer\u4f7f\u7528pull\u6a21\u5f0f\u4ecebroker\u8ba2\u9605\u5e76\u6d88\u8d39\u6d88\u606f\u3002<\/p>\n<h3>Push vs. Pull<\/h3>\n<div>\n<p>\u4f5c\u4e3a\u4e00\u4e2amessaging system\uff0cKafka\u9075\u5faa\u4e86\u4f20\u7edf\u7684\u65b9\u5f0f\uff0c\u9009\u62e9\u7531producer\u5411broker push\u6d88\u606f\u5e76\u7531consumer\u4ecebroker pull\u6d88\u606f\u3002\u4e00\u4e9blogging-centric system\uff0c\u6bd4\u5982Facebook\u7684\u00a0<a href=\"https:\/\/github.com\/facebookarchive\/scribe\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">Scribe<\/a>\u00a0\u548cCloudera\u7684\u00a0<a href=\"http:\/\/flume.apache.org\/\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">Flume<\/a>\u00a0,\u91c7\u7528\u975e\u5e38\u4e0d\u540c\u7684push\u6a21\u5f0f\u3002\u4e8b\u5b9e\u4e0a\uff0cpush\u6a21\u5f0f\u548cpull\u6a21\u5f0f\u5404\u6709\u4f18\u52a3\u3002<\/p>\n<p>push\u6a21\u5f0f\u5f88\u96be\u9002\u5e94\u6d88\u8d39\u901f\u7387\u4e0d\u540c\u7684\u6d88\u8d39\u8005\uff0c\u56e0\u4e3a\u6d88\u606f\u53d1\u9001\u901f\u7387\u662f\u7531broker\u51b3\u5b9a\u7684\u3002push\u6a21\u5f0f\u7684\u76ee\u6807\u662f\u5c3d\u53ef\u80fd\u4ee5\u6700\u5feb\u901f\u5ea6\u4f20\u9012\u6d88\u606f\uff0c\u4f46\u662f\u8fd9\u6837\u5f88\u5bb9\u6613\u9020\u6210consumer\u6765\u4e0d\u53ca\u5904\u7406\u6d88\u606f\uff0c\u5178\u578b\u7684\u8868\u73b0\u5c31\u662f\u62d2\u7edd\u670d\u52a1\u4ee5\u53ca\u7f51\u7edc\u62e5\u585e\u3002\u800cpull\u6a21\u5f0f\u5219\u53ef\u4ee5\u6839\u636econsumer\u7684\u6d88\u8d39\u80fd\u529b\u4ee5\u9002\u5f53\u7684\u901f\u7387\u6d88\u8d39\u6d88\u606f\u3002<\/p>\n<\/div>\n<h3>Topic &amp; Partition<\/h3>\n<div>\n<p>Topic\u5728\u903b\u8f91\u4e0a\u53ef\u4ee5\u88ab\u8ba4\u4e3a\u662f\u4e00\u4e2a\u5728\u7684queue\uff0c\u6bcf\u6761\u6d88\u8d39\u90fd\u5fc5\u987b\u6307\u5b9a\u5b83\u7684topic\uff0c\u53ef\u4ee5\u7b80\u5355\u7406\u89e3\u4e3a\u5fc5\u987b\u6307\u660e\u628a\u8fd9\u6761\u6d88\u606f\u653e\u8fdb\u54ea\u4e2aqueue\u91cc\u3002\u4e3a\u4e86\u4f7f\u5f97Kafka\u7684\u541e\u5410\u7387\u53ef\u4ee5\u6c34\u5e73\u6269\u5c55\uff0c\u7269\u7406\u4e0a\u628atopic\u5206\u6210\u4e00\u4e2a\u6216\u591a\u4e2apartition\uff0c\u6bcf\u4e2apartition\u5728\u7269\u7406\u4e0a\u5bf9\u5e94\u4e00\u4e2a\u6587\u4ef6\u5939\uff0c\u8be5\u6587\u4ef6\u5939\u4e0b\u5b58\u50a8\u8fd9\u4e2apartition\u7684\u6240\u6709\u6d88\u606f\u548c\u7d22\u5f15\u6587\u4ef6\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1495\" rel=\"attachment wp-att-1495\"><img loading=\"lazy\" class=\"alignnone size-full wp-image-1495\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/2-1.png\" alt=\"\" width=\"180\" height=\"114\" \/><\/a><br \/>\n\u6bcf\u4e2a\u65e5\u5fd7\u6587\u4ef6\u90fd\u662f\u201clog entries\u201d\u5e8f\u5217\uff0c\u6bcf\u4e00\u4e2a\u00a0<code>log entry<\/code>\u00a0\u5305\u542b\u4e00\u4e2a4\u5b57\u8282\u6574\u578b\u6570\uff08\u503c\u4e3aN\uff09\uff0c\u5176\u540e\u8ddfN\u4e2a\u5b57\u8282\u7684\u6d88\u606f\u4f53\u3002\u6bcf\u6761\u6d88\u606f\u90fd\u6709\u4e00\u4e2a\u5f53\u524dpartition\u4e0b\u552f\u4e00\u768464\u5b57\u8282\u7684offset\uff0c\u5b83\u6307\u660e\u4e86\u8fd9\u6761\u6d88\u606f\u7684\u8d77\u59cb\u4f4d\u7f6e\u3002\u78c1\u76d8\u4e0a\u5b58\u50a8\u7684\u6d88\u8d39\u683c\u5f0f\u5982\u4e0b\uff1a<\/p>\n<p>message length \uff1a 4 bytes (value: 1+4+n)<\/p>\n<p>\u201cmagic\u201d value \uff1a 1 byte<\/p>\n<p>crc \uff1a 4 bytes<\/p>\n<p>payload \uff1a n bytes<\/p>\n<p>\u8fd9\u4e2a\u201clog entries\u201d\u5e76\u975e\u7531\u4e00\u4e2a\u6587\u4ef6\u6784\u6210\uff0c\u800c\u662f\u5206\u6210\u591a\u4e2asegment\uff0c\u6bcf\u4e2asegment\u540d\u4e3a\u8be5segment\u7b2c\u4e00\u6761\u6d88\u606f\u7684offset\u548c\u201c.kafka\u201d\u7ec4\u6210\u3002\u53e6\u5916\u4f1a\u6709\u4e00\u4e2a\u7d22\u5f15\u6587\u4ef6\uff0c\u5b83\u6807\u660e\u4e86\u6bcf\u4e2asegment\u4e0b\u5305\u542b\u7684\u00a0<code>log entry<\/code>\u00a0\u7684offset\u8303\u56f4\uff0c\u5982\u4e0b\u56fe\u6240\u793a\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1496\" rel=\"attachment wp-att-1496\"><img loading=\"lazy\" class=\"alignnone size-full wp-image-1496\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/3-1.png\" alt=\"\" width=\"560\" height=\"513\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/3-1.png 560w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/3-1-300x275.png 300w\" sizes=\"(max-width: 560px) 100vw, 560px\" \/><\/a><br \/>\n\u56e0\u4e3a\u6bcf\u6761\u6d88\u606f\u90fd\u88abappend\u5230\u8be5partition\u4e2d\uff0c\u662f\u987a\u5e8f\u5199\u78c1\u76d8\uff0c\u56e0\u6b64\u6548\u7387\u975e\u5e38\u9ad8\uff08\u7ecf\u9a8c\u8bc1\uff0c\u987a\u5e8f\u5199\u78c1\u76d8\u6548\u7387\u6bd4\u968f\u673a\u5199\u5185\u5b58\u8fd8\u8981\u9ad8\uff0c\u8fd9\u662fKafka\u9ad8\u541e\u5410\u7387\u7684\u4e00\u4e2a\u5f88\u91cd\u8981\u7684\u4fdd\u8bc1\uff09\u3002<\/p>\n<p><img loading=\"lazy\" class=\"alignnone size-full wp-image-1497\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/4-1.png\" alt=\"\" width=\"600\" height=\"293\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/4-1.png 600w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/4-1-300x147.png 300w\" sizes=\"(max-width: 600px) 100vw, 600px\" \/><\/p>\n<\/div>\n<div>\u6bcf\u4e00\u6761\u6d88\u606f\u88ab\u53d1\u9001\u5230broker\u65f6\uff0c\u4f1a\u6839\u636eparitition\u89c4\u5219\u9009\u62e9\u88ab\u5b58\u50a8\u5230\u54ea\u4e00\u4e2apartition\u3002\u5982\u679cpartition\u89c4\u5219\u8bbe\u7f6e\u7684\u5408\u7406\uff0c\u6240\u6709\u6d88\u606f\u53ef\u4ee5\u5747\u5300\u5206\u5e03\u5230\u4e0d\u540c\u7684partition\u91cc\uff0c\u8fd9\u6837\u5c31\u5b9e\u73b0\u4e86\u6c34\u5e73\u6269\u5c55\u3002\uff08\u5982\u679c\u4e00\u4e2atopic\u5bf9\u5e94\u4e00\u4e2a\u6587\u4ef6\uff0c\u90a3\u8fd9\u4e2a\u6587\u4ef6\u6240\u5728\u7684\u673a\u5668I\/O\u5c06\u4f1a\u6210\u4e3a\u8fd9\u4e2atopic\u7684\u6027\u80fd\u74f6\u9888\uff0c\u800cpartition\u89e3\u51b3\u4e86\u8fd9\u4e2a\u95ee\u9898\uff09\u3002\u5728\u521b\u5efatopic\u65f6\u53ef\u4ee5\u5728\u00a0<code>$KAFKA_HOME\/config\/server.properties<\/code>\u00a0\u4e2d\u6307\u5b9a\u8fd9\u4e2apartition\u7684\u6570\u91cf(\u5982\u4e0b\u6240\u793a)\uff0c\u5f53\u7136\u4e5f\u53ef\u4ee5\u5728topic\u521b\u5efa\u4e4b\u540e\u53bb\u4fee\u6539parition\u6570\u91cf\u3002<\/div>\n<pre class=\"prettyprint hljs makefile\"><code><span class=\"comment\"><span class=\"hljs-comment\"># The default number of log partitions per topic. More partitions allow greater<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># parallelism for consumption, but this will also result in more files across<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># the brokers.<\/span><\/span>\r\nnum.<span class=\"variable\">partitions=<\/span><span class=\"number\">3<\/span>\r\n<\/code><\/pre>\n<p>\u5728\u53d1\u9001\u4e00\u6761\u6d88\u606f\u65f6\uff0c\u53ef\u4ee5\u6307\u5b9a\u8fd9\u6761\u6d88\u606f\u7684key\uff0cproducer\u6839\u636e\u8fd9\u4e2akey\u548cpartition\u673a\u5236\u6765\u5224\u65ad\u5c06\u8fd9\u6761\u6d88\u606f\u53d1\u9001\u5230\u54ea\u4e2aparition\u3002paritition\u673a\u5236\u53ef\u4ee5\u901a\u8fc7\u6307\u5b9aproducer\u7684paritition. class\u8fd9\u4e00\u53c2\u6570\u6765\u6307\u5b9a\uff0c\u8be5class\u5fc5\u987b\u5b9e\u73b0\u00a0<code>kafka.producer.Partitioner<\/code>\u00a0\u63a5\u53e3\u3002\u672c\u4f8b\u4e2d\u5982\u679ckey\u53ef\u4ee5\u88ab\u89e3\u6790\u4e3a\u6574\u6570\u5219\u5c06\u5bf9\u5e94\u7684\u6574\u6570\u4e0epartition\u603b\u6570\u53d6\u4f59\uff0c\u8be5\u6d88\u606f\u4f1a\u88ab\u53d1\u9001\u5230\u8be5\u6570\u5bf9\u5e94\u7684partition\u3002\uff08\u6bcf\u4e2aparition\u90fd\u4f1a\u6709\u4e2a\u5e8f\u53f7\uff09<\/p>\n<pre class=\"prettyprint hljs java\"><code><span class=\"keyword\"><span class=\"hljs-keyword\">import<\/span><\/span> kafka.producer.Partitioner;\r\n<span class=\"keyword\"><span class=\"hljs-keyword\">import<\/span><\/span> kafka.utils.VerifiableProperties;\r\n<span class=\"keyword\"><span class=\"hljs-keyword\">public<\/span><\/span> <span class=\"class\"><span class=\"keyword\"><span class=\"hljs-class\"><span class=\"hljs-keyword\">class<\/span><\/span><\/span> <span class=\"title\"><span class=\"hljs-class\"><span class=\"hljs-title\">JasonPartitioner<\/span><\/span><\/span><span class=\"hljs-class\">&lt;<\/span><span class=\"title\"><span class=\"hljs-class\"><span class=\"hljs-title\">T<\/span><\/span><\/span><span class=\"hljs-class\">&gt; <\/span><span class=\"keyword\"><span class=\"hljs-class\"><span class=\"hljs-keyword\">implements<\/span><\/span><\/span> <span class=\"title\"><span class=\"hljs-class\"><span class=\"hljs-title\">Partitioner<\/span><\/span><\/span> <\/span>{\r\n\t<span class=\"keyword\"><span class=\"hljs-function\"><span class=\"hljs-keyword\">public<\/span><\/span><\/span> <span class=\"title\"><span class=\"hljs-function\"><span class=\"hljs-title\">JasonPartitioner<\/span><\/span><\/span><span class=\"hljs-function\"><span class=\"hljs-params\">(VerifiableProperties verifiableProperties)<\/span> <\/span>{}\r\n\t<span class=\"annotation\"><span class=\"hljs-meta\">@Override<\/span><\/span>\r\n\t<span class=\"keyword\"><span class=\"hljs-function\"><span class=\"hljs-keyword\">public<\/span><\/span><\/span> <span class=\"keyword\"><span class=\"hljs-function\"><span class=\"hljs-keyword\">int<\/span><\/span><\/span> <span class=\"title\"><span class=\"hljs-function\"><span class=\"hljs-title\">partition<\/span><\/span><\/span><span class=\"hljs-function\"><span class=\"hljs-params\">(Object key, <\/span><\/span><span class=\"keyword\"><span class=\"hljs-function\"><span class=\"hljs-params\"><span class=\"hljs-keyword\">int<\/span><\/span><\/span><\/span><span class=\"hljs-function\"><span class=\"hljs-params\"> numPartitions)<\/span> <\/span>{\r\n\t\t<span class=\"keyword\"><span class=\"hljs-keyword\">try<\/span><\/span> {\r\n\t\t\t<span class=\"keyword\"><span class=\"hljs-keyword\">int<\/span><\/span> partitionNum = Integer.parseInt((String) key);\r\n\t\t\t<span class=\"keyword\"><span class=\"hljs-keyword\">return<\/span><\/span> Math.abs(Integer.parseInt((String) key) % numPartitions);\r\n\t\t} <span class=\"keyword\"><span class=\"hljs-keyword\">catch<\/span><\/span> (Exception e) {\r\n\t\t\t<span class=\"keyword\"><span class=\"hljs-keyword\">return<\/span><\/span> Math.abs(key.hashCode() % numPartitions);\r\n\t\t}\r\n\t}\r\n}\r\n<\/code><\/pre>\n<p>\u5982\u679c\u5c06\u4e0a\u4f8b\u4e2d\u7684class\u4f5c\u4e3apartition.class\uff0c\u5e76\u901a\u8fc7\u5982\u4e0b\u4ee3\u7801\u53d1\u900120\u6761\u6d88\u606f\uff08key\u5206\u522b\u4e3a0\uff0c1\uff0c2\uff0c3\uff09\u81f3topic2\uff08\u5305\u542b4\u4e2apartition\uff09\u3002<\/p>\n<pre class=\"prettyprint hljs dart\"><code><span class=\"keyword\">public<\/span> <span class=\"keyword\"><span class=\"hljs-keyword\">void<\/span><\/span> <span class=\"title\">sendMessage<\/span>() <span class=\"keyword\">throws<\/span> InterruptedException{\r\n\u3000\u3000<span class=\"keyword\"><span class=\"hljs-keyword\">for<\/span><\/span>(<span class=\"keyword\"><span class=\"hljs-built_in\">int<\/span><\/span> i = <span class=\"number\"><span class=\"hljs-number\">1<\/span><\/span>; i &lt;= <span class=\"number\"><span class=\"hljs-number\">5<\/span><\/span>; i++){\r\n\u3000\u3000      <span class=\"hljs-built_in\">List<\/span> messageList = <span class=\"keyword\"><span class=\"hljs-keyword\">new<\/span><\/span> ArrayList&lt;KeyedMessage&lt;<span class=\"hljs-built_in\">String<\/span>, <span class=\"hljs-built_in\">String<\/span>&gt;&gt;();\r\n\u3000\u3000      <span class=\"keyword\"><span class=\"hljs-keyword\">for<\/span><\/span>(<span class=\"keyword\"><span class=\"hljs-built_in\">int<\/span><\/span> j = <span class=\"number\"><span class=\"hljs-number\">0<\/span><\/span>; j &lt; <span class=\"number\"><span class=\"hljs-number\">4<\/span><\/span>; j++\uff09{\r\n\u3000\u3000          messageList.add(<span class=\"keyword\"><span class=\"hljs-keyword\">new<\/span><\/span> KeyedMessage&lt;<span class=\"hljs-built_in\">String<\/span>, <span class=\"hljs-built_in\">String<\/span>&gt;(<span class=\"string\"><span class=\"hljs-string\">\"topic2\"<\/span><\/span>, j+<span class=\"string\"><span class=\"hljs-string\">\"\"<\/span><\/span>, <span class=\"string\"><span class=\"hljs-string\">\"The \"<\/span><\/span> + i + <span class=\"string\"><span class=\"hljs-string\">\" message for key \"<\/span><\/span> + j));\r\n\u3000\u3000      }\r\n\u3000\u3000      producer.send(messageList);\r\n    }\r\n\u3000\u3000producer.close();\r\n}\r\n<\/code><\/pre>\n<div>\n<p>\u5219key\u76f8\u540c\u7684\u6d88\u606f\u4f1a\u88ab\u53d1\u9001\u5e76\u5b58\u50a8\u5230\u540c\u4e00\u4e2apartition\u91cc\uff0c\u800c\u4e14key\u7684\u5e8f\u53f7\u6b63\u597d\u548cpartition\u5e8f\u53f7\u76f8\u540c\u3002\uff08partition\u5e8f\u53f7\u4ece0\u5f00\u59cb\uff0c\u672c\u4f8b\u4e2d\u7684key\u4e5f\u6b63\u597d\u4ece0\u5f00\u59cb\uff09\u3002\u5982\u4e0b\u56fe\u6240\u793a\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1498\" rel=\"attachment wp-att-1498\"><img loading=\"lazy\" class=\"alignnone size-full wp-image-1498\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/5-1.png\" alt=\"\" width=\"867\" height=\"632\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/5-1.png 867w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/5-1-300x219.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/5-1-768x560.png 768w\" sizes=\"(max-width: 867px) 100vw, 867px\" \/><\/a>\u5bf9\u4e8e\u4f20\u7edf\u7684message queue\u800c\u8a00\uff0c\u4e00\u822c\u4f1a\u5220\u9664\u5df2\u7ecf\u88ab\u6d88\u8d39\u7684\u6d88\u606f\uff0c\u800cKafka\u96c6\u7fa4\u4f1a\u4fdd\u7559\u6240\u6709\u7684\u6d88\u606f\uff0c\u65e0\u8bba\u5176\u88ab\u6d88\u8d39\u4e0e\u5426\u3002\u5f53\u7136\uff0c\u56e0\u4e3a\u78c1\u76d8\u9650\u5236\uff0c\u4e0d\u53ef\u80fd\u6c38\u4e45\u4fdd\u7559\u6240\u6709\u6570\u636e\uff08\u5b9e\u9645\u4e0a\u4e5f\u6ca1\u5fc5\u8981\uff09\uff0c\u56e0\u6b64Kafka\u63d0\u4f9b\u4e24\u79cd\u7b56\u7565\u53bb\u5220\u9664\u65e7\u6570\u636e\u3002\u4e00\u662f\u57fa\u4e8e\u65f6\u95f4\uff0c\u4e8c\u662f\u57fa\u4e8epartition\u6587\u4ef6\u5927\u5c0f\u3002\u4f8b\u5982\u53ef\u4ee5\u901a\u8fc7\u914d\u7f6e\u00a0<code>$KAFKA_HOME\/config\/server.properties<\/code>\u00a0\uff0c\u8ba9Kafka\u5220\u9664\u4e00\u5468\u524d\u7684\u6570\u636e\uff0c\u4e5f\u53ef\u901a\u8fc7\u914d\u7f6e\u8ba9Kafka\u5728partition\u6587\u4ef6\u8d85\u8fc71GB\u65f6\u5220\u9664\u65e7\u6570\u636e\uff0c\u5982\u4e0b\u6240\u793a\u3002<\/p>\n<\/div>\n<pre class=\"prettyprint hljs coffeescript\"><code>\u3000\u3000<span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##### Log Retention Policy ###<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The following configurations control the disposal of log segments. The policy can<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># be set to delete segments after a period of time, or after a given size has accumulated.<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># A segment will be deleted whenever *either* of these criteria are met. Deletion always happens<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># from the end of the log.<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The minimum age of a log file to be eligible for deletion<\/span><\/span>\r\nlog.retention.hours=<span class=\"number\"><span class=\"hljs-number\">168<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># A size-based retention policy for logs. Segments are pruned from the log as long as the remaining<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># segments don't drop below log.retention.bytes.<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\">#log.retention.bytes=1073741824<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The maximum size of a log segment file. When this size is reached a new log segment will be created.<\/span><\/span>\r\nlog.segment.bytes=<span class=\"number\"><span class=\"hljs-number\">1073741824<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The interval at which log segments are checked to see if they can be deleted according<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># to the retention policies<\/span><\/span>\r\nlog.retention.check.interval.ms=<span class=\"number\"><span class=\"hljs-number\">300000<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># By default the log cleaner is disabled and the log retention policy will default to <\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\">#just delete segments after their retention expires.<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># If log.cleaner.enable=true is set the cleaner will be enabled and individual logs <\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\">#can then be marked for log compaction.<\/span><\/span>\r\nlog.cleaner.enable=<span class=\"literal\"><span class=\"hljs-literal\">false<\/span><\/span>\r\n<\/code><\/pre>\n<p>\u8fd9\u91cc\u8981\u6ce8\u610f\uff0c\u56e0\u4e3aKafka\u8bfb\u53d6\u7279\u5b9a\u6d88\u606f\u7684\u65f6\u95f4\u590d\u6742\u5ea6\u4e3aO(1)\uff0c\u5373\u4e0e\u6587\u4ef6\u5927\u5c0f\u65e0\u5173\uff0c\u6240\u4ee5\u8fd9\u91cc\u5220\u9664\u6587\u4ef6\u4e0eKafka\u6027\u80fd\u65e0\u5173\uff0c\u9009\u62e9\u600e\u6837\u7684\u5220\u9664\u7b56\u7565\u53ea\u4e0e\u78c1\u76d8\u4ee5\u53ca\u5177\u4f53\u7684\u9700\u6c42\u6709\u5173\u3002\u53e6\u5916\uff0cKafka\u4f1a\u4e3a\u6bcf\u4e00\u4e2aconsumer group\u4fdd\u7559\u4e00\u4e9bmetadata\u4fe1\u606f\u2014\u5f53\u524d\u6d88\u8d39\u7684\u6d88\u606f\u7684position\uff0c\u4e5f\u5373offset\u3002\u8fd9\u4e2aoffset\u7531consumer\u63a7\u5236\u3002\u6b63\u5e38\u60c5\u51b5\u4e0bconsumer\u4f1a\u5728\u6d88\u8d39\u5b8c\u4e00\u6761\u6d88\u606f\u540e\u7ebf\u6027\u589e\u52a0\u8fd9\u4e2aoffset\u3002\u5f53\u7136\uff0cconsumer\u4e5f\u53ef\u5c06offset\u8bbe\u6210\u4e00\u4e2a\u8f83\u5c0f\u7684\u503c\uff0c\u91cd\u65b0\u6d88\u8d39\u4e00\u4e9b\u6d88\u606f\u3002\u56e0\u4e3aoffet\u7531consumer\u63a7\u5236\uff0c\u6240\u4ee5Kafka broker\u662f\u65e0\u72b6\u6001\u7684\uff0c\u5b83\u4e0d\u9700\u8981\u6807\u8bb0\u54ea\u4e9b\u6d88\u606f\u88ab\u54ea\u4e9bconsumer\u8fc7\uff0c\u4e0d\u9700\u8981\u901a\u8fc7broker\u53bb\u4fdd\u8bc1\u540c\u4e00\u4e2aconsumer group\u53ea\u6709\u4e00\u4e2aconsumer\u80fd\u6d88\u8d39\u67d0\u4e00\u6761\u6d88\u606f\uff0c\u56e0\u6b64\u4e5f\u5c31\u4e0d\u9700\u8981\u9501\u673a\u5236\uff0c\u8fd9\u4e5f\u4e3aKafka\u7684\u9ad8\u541e\u5410\u7387\u63d0\u4f9b\u4e86\u6709\u529b\u4fdd\u969c\u3002<\/p>\n<h3>Replication &amp; Leader election<\/h3>\n<p>Kafka\u4ece0.8\u5f00\u59cb\u63d0\u4f9bpartition\u7ea7\u522b\u7684replication\uff0creplication\u7684\u6570\u91cf\u53ef\u5728\u00a0<code>$KAFKA_HOME\/config\/server.properties<\/code>\u00a0\u4e2d\u914d\u7f6e\u3002<\/p>\n<pre class=\"prettyprint hljs haskell\"><code><span class=\"default\"><span class=\"keyword\"><span class=\"hljs-keyword\">default<\/span><\/span>.replication.factor = 1<\/span>\r\n<\/code><\/pre>\n<div>\n<p>\u8be5 Replication\u4e0eleader election\u914d\u5408\u63d0\u4f9b\u4e86\u81ea\u52a8\u7684failover\u673a\u5236\u3002replication\u5bf9Kafka\u7684\u541e\u5410\u7387\u662f\u6709\u4e00\u5b9a\u5f71\u54cd\u7684\uff0c\u4f46\u6781\u5927\u7684\u589e\u5f3a\u4e86\u53ef\u7528\u6027\u3002\u9ed8\u8ba4\u60c5\u51b5\u4e0b\uff0cKafka\u7684replication\u6570\u91cf\u4e3a1\u3002 \u6bcf\u4e2apartition\u90fd\u6709\u4e00\u4e2a\u552f\u4e00\u7684leader\uff0c\u6240\u6709\u7684\u8bfb\u5199\u64cd\u4f5c\u90fd\u5728leader\u4e0a\u5b8c\u6210\uff0cleader\u6279\u91cf\u4eceleader\u4e0apull\u6570\u636e\u3002\u4e00\u822c\u60c5\u51b5\u4e0bpartition\u7684\u6570\u91cf\u5927\u4e8e\u7b49\u4e8ebroker\u7684\u6570\u91cf\uff0c\u5e76\u4e14\u6240\u6709partition\u7684leader\u5747\u5300\u5206\u5e03\u5728broker\u4e0a\u3002follower\u4e0a\u7684\u65e5\u5fd7\u548c\u5176leader\u4e0a\u7684\u5b8c\u5168\u4e00\u6837\u3002<\/p>\n<p>\u548c\u5927\u90e8\u5206\u5206\u5e03\u5f0f\u7cfb\u7edf\u4e00\u6837\uff0cKakfa\u5904\u7406\u5931\u8d25\u9700\u8981\u660e\u786e\u5b9a\u4e49\u4e00\u4e2abroker\u662f\u5426alive\u3002\u5bf9\u4e8eKafka\u800c\u8a00\uff0cKafka\u5b58\u6d3b\u5305\u542b\u4e24\u4e2a\u6761\u4ef6\uff0c\u4e00\u662f\u5b83\u5fc5\u987b\u7ef4\u62a4\u4e0eZookeeper\u7684session(\u8fd9\u4e2a\u901a\u8fc7Zookeeper\u7684heartbeat\u673a\u5236\u6765\u5b9e\u73b0)\u3002\u4e8c\u662ffollower\u5fc5\u987b\u80fd\u591f\u53ca\u65f6\u5c06leader\u7684writing\u590d\u5236\u8fc7\u6765\uff0c\u4e0d\u80fd\u201c\u843d\u540e\u592a\u591a\u201d\u3002<\/p>\n<p>leader\u4f1atrack\u201cin sync\u201d\u7684node list\u3002\u5982\u679c\u4e00\u4e2afollower\u5b95\u673a\uff0c\u6216\u8005\u843d\u540e\u592a\u591a\uff0cleader\u5c06\u628a\u5b83\u4ece\u201din sync\u201d list\u4e2d\u79fb\u9664\u3002\u8fd9\u91cc\u6240\u63cf\u8ff0\u7684\u201c\u843d\u540e\u592a\u591a\u201d\u6307follower\u590d\u5236\u7684\u6d88\u606f\u843d\u540e\u4e8eleader\u540e\u7684\u6761\u6570\u8d85\u8fc7\u9884\u5b9a\u503c\uff0c\u8be5\u503c\u53ef\u5728\u00a0<code>$KAFKA_HOME\/config\/server.properties<\/code>\u00a0\u4e2d\u914d\u7f6e<\/p>\n<\/div>\n<pre class=\"prettyprint hljs tcl\"><code><span class=\"comment\"><span class=\"hljs-comment\">#If a replica falls more than this many messages behind the leader, the leader will remove the follower from ISR and treat it as dead<\/span><\/span>\r\nreplica.lag.<span class=\"built_in\">max<\/span>.messages=<span class=\"number\"><span class=\"hljs-number\">4000<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\">#If a follower hasn't sent any fetch requests for this window of time, the leader will remove the follower from ISR (in-sync replicas) and treat it as dead<\/span><\/span>\r\nreplica.lag.<span class=\"built_in\"><span class=\"hljs-keyword\">time<\/span><\/span>.<span class=\"built_in\">max<\/span>.ms=<span class=\"number\"><span class=\"hljs-number\">10000<\/span><\/span>  \r\n<\/code><\/pre>\n<div>\n<p>\u9700\u8981\u8bf4\u660e\u7684\u662f\uff0cKafka\u53ea\u89e3\u51b3\u201dfail\/recover\u201d\uff0c\u4e0d\u5904\u7406\u201cByzantine\u201d\uff08\u201c\u62dc\u5360\u5ead\u201d\uff09\u95ee\u9898\u3002<\/p>\n<p>\u4e00\u6761\u6d88\u606f\u53ea\u6709\u88ab\u201cin sync\u201d list\u91cc\u7684\u6240\u6709follower\u90fd\u4eceleader\u590d\u5236\u8fc7\u53bb\u624d\u4f1a\u88ab\u8ba4\u4e3a\u5df2\u63d0\u4ea4\u3002\u8fd9\u6837\u5c31\u907f\u514d\u4e86\u90e8\u5206\u6570\u636e\u88ab\u5199\u8fdb\u4e86leader\uff0c\u8fd8\u6ca1\u6765\u5f97\u53ca\u88ab\u4efb\u4f55follower\u590d\u5236\u5c31\u5b95\u673a\u4e86\uff0c\u800c\u9020\u6210\u6570\u636e\u4e22\u5931\uff08consumer\u65e0\u6cd5\u6d88\u8d39\u8fd9\u4e9b\u6570\u636e\uff09\u3002\u800c\u5bf9\u4e8eproducer\u800c\u8a00\uff0c\u5b83\u53ef\u4ee5\u9009\u62e9\u662f\u5426\u7b49\u5f85\u6d88\u606fcommit\uff0c\u8fd9\u53ef\u4ee5\u901a\u8fc7\u00a0<code>request.required.acks<\/code>\u00a0\u6765\u8bbe\u7f6e\u3002\u8fd9\u79cd\u673a\u5236\u786e\u4fdd\u4e86\u53ea\u8981\u201cin sync\u201d list\u6709\u4e00\u4e2a\u6216\u4ee5\u4e0a\u7684flollower\uff0c\u4e00\u6761\u88abcommit\u7684\u6d88\u606f\u5c31\u4e0d\u4f1a\u4e22\u5931\u3002<\/p>\n<p>\u8fd9\u91cc\u7684\u590d\u5236\u673a\u5236\u5373\u4e0d\u662f\u540c\u6b65\u590d\u5236\uff0c\u4e5f\u4e0d\u662f\u5355\u7eaf\u7684\u5f02\u6b65\u590d\u5236\u3002\u4e8b\u5b9e\u4e0a\uff0c\u540c\u6b65\u590d\u5236\u8981\u6c42\u201c\u6d3b\u7740\u7684\u201dfollower\u90fd\u590d\u5236\u5b8c\uff0c\u8fd9\u6761\u6d88\u606f\u624d\u4f1a\u88ab\u8ba4\u4e3acommit\uff0c\u8fd9\u79cd\u590d\u5236\u65b9\u5f0f\u6781\u5927\u7684\u5f71\u54cd\u4e86\u541e\u5410\u7387\uff08\u9ad8\u541e\u5410\u7387\u662fKafka\u975e\u5e38\u91cd\u8981\u7684\u4e00\u4e2a\u7279\u6027\uff09\u3002\u800c\u5f02\u6b65\u590d\u5236\u65b9\u5f0f\u4e0b\uff0cfollower\u5f02\u6b65\u7684\u4eceleader\u590d\u5236\u6570\u636e\uff0c\u6570\u636e\u53ea\u8981\u88ableader\u5199\u5165log\u5c31\u88ab\u8ba4\u4e3a\u5df2\u7ecfcommit\uff0c\u8fd9\u79cd\u60c5\u51b5\u4e0b\u5982\u679cfollwer\u90fd\u843d\u540e\u4e8eleader\uff0c\u800cleader\u7a81\u7136\u5b95\u673a\uff0c\u5219\u4f1a\u4e22\u5931\u6570\u636e\u3002\u800cKafka\u7684\u8fd9\u79cd\u4f7f\u7528\u201cin sync\u201d list\u7684\u65b9\u5f0f\u5219\u5f88\u597d\u7684\u5747\u8861\u4e86\u786e\u4fdd\u6570\u636e\u4e0d\u4e22\u5931\u4ee5\u53ca\u541e\u5410\u7387\u3002follower\u53ef\u4ee5\u6279\u91cf\u7684\u4eceleader\u590d\u5236\u6570\u636e\uff0c\u8fd9\u6837\u6781\u5927\u7684\u63d0\u9ad8\u590d\u5236\u6027\u80fd\uff08\u6279\u91cf\u5199\u78c1\u76d8\uff09\uff0c\u6781\u5927\u51cf\u5c11\u4e86follower\u4e0eleader\u7684\u5dee\u8ddd\uff08\u524d\u6587\u6709\u8bf4\u5230\uff0c\u53ea\u8981follower\u843d\u540eleader\u4e0d\u592a\u8fdc\uff0c\u5219\u88ab\u8ba4\u4e3a\u5728\u201cin sync\u201d list\u91cc\uff09\u3002<\/p>\n<\/div>\n<div>\n<p>\u4e0a\u6587\u8bf4\u660e\u4e86Kafka\u662f\u5982\u4f55\u505areplication\u7684\uff0c\u53e6\u5916\u4e00\u4e2a\u5f88\u91cd\u8981\u7684\u95ee\u9898\u662f\u5f53leader\u5b95\u673a\u4e86\uff0c\u600e\u6837\u5728follower\u4e2d\u9009\u4e3e\u51fa\u65b0\u7684leader\u3002\u56e0\u4e3afollower\u53ef\u80fd\u843d\u540e\u8bb8\u591a\u6216\u8005crash\u4e86\uff0c\u6240\u4ee5\u5fc5\u987b\u786e\u4fdd\u9009\u62e9\u201c\u6700\u65b0\u201d\u7684follower\u4f5c\u4e3a\u65b0\u7684leader\u3002\u4e00\u4e2a\u57fa\u672c\u7684\u539f\u5219\u5c31\u662f\uff0c\u5982\u679cleader\u4e0d\u5728\u4e86\uff0c\u65b0\u7684leader\u5fc5\u987b\u62e5\u6709\u539f\u6765\u7684leader commit\u7684\u6240\u6709\u6d88\u606f\u3002\u8fd9\u5c31\u9700\u8981\u4f5c\u4e00\u4e2a\u6298\u8877\uff0c\u5982\u679cleader\u5728\u6807\u660e\u4e00\u6761\u6d88\u606f\u88abcommit\u524d\u7b49\u5f85\u66f4\u591a\u7684follower\u786e\u8ba4\uff0c\u90a3\u5728\u5b83die\u4e4b\u540e\u5c31\u6709\u66f4\u591a\u7684follower\u53ef\u4ee5\u4f5c\u4e3a\u65b0\u7684leader\uff0c\u4f46\u8fd9\u4e5f\u4f1a\u9020\u6210\u541e\u5410\u7387\u7684\u4e0b\u964d\u3002<\/p>\n<p>\u4e00\u79cd\u975e\u5e38\u5e38\u7528\u7684\u9009\u4e3eleader\u7684\u65b9\u5f0f\u662f\u201cmajority \u7075\u79c0\u201d\uff08\u201c\u5c11\u6570\u670d\u4ece\u591a\u6570\u201d\uff09\uff0c\u4f46Kafka\u5e76\u672a\u91c7\u7528\u8fd9\u79cd\u65b9\u5f0f\u3002\u8fd9\u79cd\u6a21\u5f0f\u4e0b\uff0c\u5982\u679c\u6211\u4eec\u67092f+1\u4e2areplica\uff08\u5305\u542bleader\u548cfollower\uff09\uff0c\u90a3\u5728commit\u4e4b\u524d\u5fc5\u987b\u4fdd\u8bc1\u6709f+1\u4e2areplica\u590d\u5236\u5b8c\u6d88\u606f\uff0c\u4e3a\u4e86\u4fdd\u8bc1\u6b63\u786e\u9009\u51fa\u65b0\u7684leader\uff0cfail\u7684replica\u4e0d\u80fd\u8d85\u8fc7f\u4e2a\u3002\u56e0\u4e3a\u5728\u5269\u4e0b\u7684\u4efb\u610ff+1\u4e2areplica\u91cc\uff0c\u81f3\u5c11\u6709\u4e00\u4e2areplica\u5305\u542b\u6709\u6700\u65b0\u7684\u6240\u6709\u6d88\u606f\u3002\u8fd9\u79cd\u65b9\u5f0f\u6709\u4e2a\u5f88\u5927\u7684\u4f18\u52bf\uff0c\u7cfb\u7edf\u7684latency\u53ea\u53d6\u51b3\u4e8e\u6700\u5feb\u7684\u51e0\u53f0server\uff0c\u4e5f\u5c31\u662f\u8bf4\uff0c\u5982\u679creplication factor\u662f3\uff0c\u90a3latency\u5c31\u53d6\u51b3\u4e8e\u6700\u5feb\u7684\u90a3\u4e2afollower\u800c\u975e\u6700\u6162\u90a3\u4e2a\u3002majority vote\u4e5f\u6709\u4e00\u4e9b\u52a3\u52bf\uff0c\u4e3a\u4e86\u4fdd\u8bc1leader election\u7684\u6b63\u5e38\u8fdb\u884c\uff0c\u5b83\u6240\u80fd\u5bb9\u5fcd\u7684fail\u7684follower\u4e2a\u6570\u6bd4\u8f83\u5c11\u3002\u5982\u679c\u8981\u5bb9\u5fcd1\u4e2afollower\u6302\u6389\uff0c\u5fc5\u987b\u8981\u67093\u4e2a\u4ee5\u4e0a\u7684replica\uff0c\u5982\u679c\u8981\u5bb9\u5fcd2\u4e2afollower\u6302\u6389\uff0c\u5fc5\u987b\u8981\u67095\u4e2a\u4ee5\u4e0a\u7684replica\u3002\u4e5f\u5c31\u662f\u8bf4\uff0c\u5728\u751f\u4ea7\u73af\u5883\u4e0b\u4e3a\u4e86\u4fdd\u8bc1\u8f83\u9ad8\u7684\u5bb9\u9519\u7a0b\u5ea6\uff0c\u5fc5\u987b\u8981\u6709\u5927\u91cf\u7684replica\uff0c\u800c\u5927\u91cf\u7684replica\u53c8\u4f1a\u5728\u5927\u6570\u636e\u91cf\u4e0b\u5bfc\u81f4\u6027\u80fd\u7684\u6025\u5267\u4e0b\u964d\u3002\u8fd9\u5c31\u662f\u8fd9\u79cd\u7b97\u6cd5\u66f4\u591a\u7528\u5728\u00a0<a href=\"http:\/\/zookeeper.apache.org\/\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">Zookeeper<\/a>\u00a0\u8fd9\u79cd\u5171\u4eab\u96c6\u7fa4\u914d\u7f6e\u7684\u7cfb\u7edf\u4e2d\u800c\u5f88\u5c11\u5728\u9700\u8981\u5b58\u50a8\u5927\u91cf\u6570\u636e\u7684\u7cfb\u7edf\u4e2d\u4f7f\u7528\u7684\u539f\u56e0\u3002\u4f8b\u5982HDFS\u7684HA feature\u662f\u57fa\u4e8e\u00a0<a href=\"http:\/\/blog.cloudera.com\/blog\/2012\/10\/quorum-based-journaling-in-cdh4-1\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">majority-vote-based journal<\/a>\u00a0\uff0c\u4f46\u662f\u5b83\u7684\u6570\u636e\u5b58\u50a8\u5e76\u6ca1\u6709\u4f7f\u7528\u8fd9\u79cdexpensive\u7684\u65b9\u5f0f\u3002<\/p>\n<p>\u5b9e\u9645\u4e0a\uff0cleader election\u7b97\u6cd5\u975e\u5e38\u591a\uff0c\u6bd4\u5982Zookeper\u7684\u00a0<a href=\"http:\/\/web.stanford.edu\/class\/cs347\/reading\/zab.pdf\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">Zab<\/a>\u00a0,\u00a0<a href=\"https:\/\/ramcloud.stanford.edu\/wiki\/download\/attachments\/11370504\/raft.pdf\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">Raft<\/a>\u00a0\u548c\u00a0<a href=\"http:\/\/pmg.csail.mit.edu\/papers\/vr-revisited.pdf\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">Viewstamped Replication<\/a>\u00a0\u3002\u800cKafka\u6240\u4f7f\u7528\u7684leader election\u7b97\u6cd5\u66f4\u50cf\u5fae\u8f6f\u7684\u00a0<a href=\"http:\/\/research.microsoft.com\/apps\/pubs\/default.aspx?id=66814\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">PacificA<\/a>\u00a0\u7b97\u6cd5\u3002<\/p>\n<p>Kafka\u5728Zookeeper\u4e2d\u52a8\u6001\u7ef4\u62a4\u4e86\u4e00\u4e2aISR\uff08in-sync replicas\uff09 set\uff0c\u8fd9\u4e2aset\u91cc\u7684\u6240\u6709replica\u90fd\u8ddf\u4e0a\u4e86leader\uff0c\u53ea\u6709ISR\u91cc\u7684\u6210\u5458\u624d\u6709\u88ab\u9009\u4e3aleader\u7684\u53ef\u80fd\u3002\u5728\u8fd9\u79cd\u6a21\u5f0f\u4e0b\uff0c\u5bf9\u4e8ef+1\u4e2areplica\uff0c\u4e00\u4e2aKafka topic\u80fd\u5728\u4fdd\u8bc1\u4e0d\u4e22\u5931\u5df2\u7ecfommit\u7684\u6d88\u606f\u7684\u524d\u63d0\u4e0b\u5bb9\u5fcdf\u4e2areplica\u7684\u5931\u8d25\u3002\u5728\u5927\u591a\u6570\u4f7f\u7528\u573a\u666f\u4e2d\uff0c\u8fd9\u79cd\u6a21\u5f0f\u662f\u975e\u5e38\u6709\u5229\u7684\u3002\u4e8b\u5b9e\u4e0a\uff0c\u4e3a\u4e86\u5bb9\u5fcdf\u4e2areplica\u7684\u5931\u8d25\uff0cmajority vote\u548cISR\u5728commit\u524d\u9700\u8981\u7b49\u5f85\u7684replica\u6570\u91cf\u662f\u4e00\u6837\u7684\uff0c\u4f46\u662fISR\u9700\u8981\u7684\u603b\u7684replica\u7684\u4e2a\u6570\u51e0\u4e4e\u662fmajority vote\u7684\u4e00\u534a\u3002<\/p>\n<p>\u867d\u7136majority vote\u4e0eISR\u76f8\u6bd4\u6709\u4e0d\u9700\u7b49\u5f85\u6700\u6162\u7684server\u8fd9\u4e00\u4f18\u52bf\uff0c\u4f46\u662fKafka\u4f5c\u8005\u8ba4\u4e3aKafka\u53ef\u4ee5\u901a\u8fc7producer\u9009\u62e9\u662f\u5426\u88abcommit\u963b\u585e\u6765\u6539\u5584\u8fd9\u4e00\u95ee\u9898\uff0c\u5e76\u4e14\u8282\u7701\u4e0b\u6765\u7684replica\u548c\u78c1\u76d8\u4f7f\u5f97ISR\u6a21\u5f0f\u4ecd\u7136\u503c\u5f97\u3002<\/p>\n<\/div>\n<p>\u4e0a\u6587\u63d0\u5230\uff0c\u5728ISR\u4e2d\u81f3\u5c11\u6709\u4e00\u4e2afollower\u65f6\uff0cKafka\u53ef\u4ee5\u786e\u4fdd\u5df2\u7ecfcommit\u7684\u6570\u636e\u4e0d\u4e22\u5931\uff0c\u4f46\u5982\u679c\u67d0\u4e00\u4e2apartition\u7684\u6240\u6709replica\u90fd\u6302\u4e86\uff0c\u5c31\u65e0\u6cd5\u4fdd\u8bc1\u6570\u636e\u4e0d\u4e22\u5931\u4e86\u3002\u8fd9\u79cd\u60c5\u51b5\u4e0b\u6709\u4e24\u79cd\u53ef\u884c\u7684\u65b9\u6848\uff1a<\/p>\n<ul>\n<li>\u7b49\u5f85ISR\u4e2d\u7684\u4efb\u4e00\u4e2areplica\u201c\u6d3b\u201d\u8fc7\u6765\uff0c\u5e76\u4e14\u9009\u5b83\u4f5c\u4e3aleader<\/li>\n<li>\u9009\u62e9\u7b2c\u4e00\u4e2a\u201c\u6d3b\u201d\u8fc7\u6765\u7684replica\uff08\u4e0d\u4e00\u5b9a\u662fISR\u4e2d\u7684\uff09\u4f5c\u4e3aleader<\/li>\n<\/ul>\n<p>\u8fd9\u5c31\u9700\u8981\u5728\u53ef\u7528\u6027\u548c\u4e00\u81f4\u6027\u5f53\u4e2d\u4f5c\u51fa\u4e00\u4e2a\u7b80\u5355\u7684\u5e73\u8861\u3002\u5982\u679c\u4e00\u5b9a\u8981\u7b49\u5f85ISR\u4e2d\u7684replica\u201c\u6d3b\u201d\u8fc7\u6765\uff0c\u90a3\u4e0d\u53ef\u7528\u7684\u65f6\u95f4\u5c31\u53ef\u80fd\u4f1a\u76f8\u5bf9\u8f83\u957f\u3002\u800c\u4e14\u5982\u679cISR\u4e2d\u7684\u6240\u6709replica\u90fd\u65e0\u6cd5\u201c\u6d3b\u201d\u8fc7\u6765\u4e86\uff0c\u6216\u8005\u6570\u636e\u90fd\u4e22\u5931\u4e86\uff0c\u8fd9\u4e2apartition\u5c06\u6c38\u8fdc\u4e0d\u53ef\u7528\u3002\u9009\u62e9\u7b2c\u4e00\u4e2a\u201c\u6d3b\u201d\u8fc7\u6765\u7684replica\u4f5c\u4e3aleader\uff0c\u800c\u8fd9\u4e2areplica\u4e0d\u662fISR\u4e2d\u7684replica\uff0c\u90a3\u5373\u4f7f\u5b83\u5e76\u4e0d\u4fdd\u8bc1\u5df2\u7ecf\u5305\u542b\u4e86\u6240\u6709\u5df2commit\u7684\u6d88\u606f\uff0c\u5b83\u4e5f\u4f1a\u6210\u4e3aleader\u800c\u4f5c\u4e3aconsumer\u7684\u6570\u636e\u6e90\uff08\u524d\u6587\u6709\u8bf4\u660e\uff0c\u6240\u6709\u8bfb\u5199\u90fd\u7531leader\u5b8c\u6210\uff09\u3002Kafka0.8.*\u4f7f\u7528\u4e86\u7b2c\u4e8c\u79cd\u65b9\u5f0f\u3002\u6839\u636eKafka\u7684\u6587\u6863\uff0c\u5728\u4ee5\u540e\u7684\u7248\u672c\u4e2d\uff0cKafka\u652f\u6301\u7528\u6237\u901a\u8fc7\u914d\u7f6e\u9009\u62e9\u8fd9\u4e24\u79cd\u65b9\u5f0f\u4e2d\u7684\u4e00\u79cd\uff0c\u4ece\u800c\u6839\u636e\u4e0d\u540c\u7684\u4f7f\u7528\u573a\u666f\u9009\u62e9\u9ad8\u53ef\u7528\u6027\u8fd8\u662f\u5f3a\u4e00\u81f4\u6027\u3002<\/p>\n<div>\n<p>\u4e0a\u6587\u8bf4\u660e\u4e86\u4e00\u4e2aparition\u7684replication\u8fc7\u7a0b\uff0c\u7136\u5c14Kafka\u96c6\u7fa4\u9700\u8981\u7ba1\u7406\u6210\u767e\u4e0a\u5343\u4e2apartition\uff0cKafka\u901a\u8fc7round-robin\u7684\u65b9\u5f0f\u6765\u5e73\u8861partition\u4ece\u800c\u907f\u514d\u5927\u91cfpartition\u96c6\u4e2d\u5728\u4e86\u5c11\u6570\u51e0\u4e2a\u8282\u70b9\u4e0a\u3002\u540c\u65f6Kafka\u4e5f\u9700\u8981\u5e73\u8861leader\u7684\u5206\u5e03\uff0c\u5c3d\u53ef\u80fd\u7684\u8ba9\u6240\u6709partition\u7684leader\u5747\u5300\u5206\u5e03\u5728\u4e0d\u540cbroker\u4e0a\u3002\u53e6\u4e00\u65b9\u9762\uff0c\u4f18\u5316leadership election\u7684\u8fc7\u7a0b\u4e5f\u662f\u5f88\u91cd\u8981\u7684\uff0c\u6bd5\u7adf\u8fd9\u6bb5\u65f6\u95f4\u76f8\u5e94\u7684partition\u5904\u4e8e\u4e0d\u53ef\u7528\u72b6\u6001\u3002\u4e00\u79cd\u7b80\u5355\u7684\u5b9e\u73b0\u662f\u6682\u505c\u5b95\u673a\u7684broker\u4e0a\u7684\u6240\u6709partition\uff0c\u5e76\u4e3a\u4e4b\u9009\u4e3eleader\u3002\u5b9e\u9645\u4e0a\uff0cKafka\u9009\u4e3e\u4e00\u4e2abroker\u4f5c\u4e3acontroller\uff0c\u8fd9\u4e2acontroller\u901a\u8fc7watch Zookeeper\u68c0\u6d4b\u6240\u6709\u7684broker failure\uff0c\u5e76\u8d1f\u8d23\u4e3a\u6240\u6709\u53d7\u5f71\u54cd\u7684parition\u9009\u4e3eleader\uff0c\u518d\u5c06\u76f8\u5e94\u7684leader\u8c03\u6574\u547d\u4ee4\u53d1\u9001\u81f3\u53d7\u5f71\u54cd\u7684broker\uff0c\u8fc7\u7a0b\u5982\u4e0b\u56fe\u6240\u793a\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1499\" rel=\"attachment wp-att-1499\"><img loading=\"lazy\" class=\"alignnone size-full wp-image-1499\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/6-1.png\" alt=\"\" width=\"800\" height=\"528\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/6-1.png 800w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/6-1-300x198.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/6-1-768x507.png 768w\" sizes=\"(max-width: 800px) 100vw, 800px\" \/><\/a><\/p>\n<\/div>\n<div>\n<p>\u8fd9\u6837\u505a\u7684\u597d\u5904\u662f\uff0c\u53ef\u4ee5\u6279\u91cf\u7684\u901a\u77e5leadership\u7684\u53d8\u5316\uff0c\u4ece\u800c\u4f7f\u5f97\u9009\u4e3e\u8fc7\u7a0b\u6210\u672c\u66f4\u4f4e\uff0c\u5c24\u5176\u5bf9\u5927\u91cf\u7684partition\u800c\u8a00\u3002\u5982\u679ccontroller\u5931\u8d25\u4e86\uff0c\u5e78\u5b58\u7684\u6240\u6709broker\u90fd\u4f1a\u5c1d\u8bd5\u5728Zookeeper\u4e2d\u521b\u5efa\/controller-&gt;{this broker id}\uff0c\u5982\u679c\u521b\u5efa\u6210\u529f\uff08\u53ea\u53ef\u80fd\u6709\u4e00\u4e2a\u521b\u5efa\u6210\u529f\uff09\uff0c\u5219\u8be5broker\u4f1a\u6210\u4e3acontroller\uff0c\u82e5\u521b\u5efa\u4e0d\u6210\u529f\uff0c\u5219\u8be5broker\u4f1a\u7b49\u5f85\u65b0controller\u7684\u547d\u4ee4\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1500\" rel=\"attachment wp-att-1500\"><img loading=\"lazy\" class=\"alignnone size-full wp-image-1500\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/7.png\" alt=\"\" width=\"800\" height=\"456\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/7.png 800w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/7-300x171.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/7-768x438.png 768w\" sizes=\"(max-width: 800px) 100vw, 800px\" \/><\/a><\/p>\n<\/div>\n<h3>Consumer group<\/h3>\n<div>\n<p>\uff08\u672c\u8282\u6240\u6709\u63cf\u8ff0\u90fd\u662f\u57fa\u4e8econsumer hight level API\u800c\u975elow level API\uff09\u3002<\/p>\n<p>\u6bcf\u4e00\u4e2aconsumer\u5b9e\u4f8b\u90fd\u5c5e\u4e8e\u4e00\u4e2aconsumer group\uff0c\u6bcf\u4e00\u6761\u6d88\u606f\u53ea\u4f1a\u88ab\u540c\u4e00\u4e2aconsumer group\u91cc\u7684\u4e00\u4e2aconsumer\u5b9e\u4f8b\u6d88\u8d39\u3002\uff08\u4e0d\u540cconsumer group\u53ef\u4ee5\u540c\u65f6\u6d88\u8d39\u540c\u4e00\u6761\u6d88\u606f\uff09<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1501\" rel=\"attachment wp-att-1501\"><img loading=\"lazy\" class=\"alignnone size-full wp-image-1501\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/8.png\" alt=\"\" width=\"800\" height=\"497\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/8.png 800w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/8-300x186.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/8-768x477.png 768w\" sizes=\"(max-width: 800px) 100vw, 800px\" \/><\/a><\/p>\n<\/div>\n<div>\n<p>\u5f88\u591a\u4f20\u7edf\u7684message queue\u90fd\u4f1a\u5728\u6d88\u606f\u88ab\u6d88\u8d39\u5b8c\u540e\u5c06\u6d88\u606f\u5220\u9664\uff0c\u4e00\u65b9\u9762\u907f\u514d\u91cd\u590d\u6d88\u8d39\uff0c\u53e6\u4e00\u65b9\u9762\u53ef\u4ee5\u4fdd\u8bc1queue\u7684\u957f\u5ea6\u6bd4\u8f83\u5c11\uff0c\u63d0\u9ad8\u6548\u7387\u3002\u800c\u5982\u4e0a\u6587\u6240\u5c06\uff0cKafka\u5e76\u4e0d\u5220\u9664\u5df2\u6d88\u8d39\u7684\u6d88\u606f\uff0c\u4e3a\u4e86\u5b9e\u73b0\u4f20\u7edfmessage queue\u6d88\u606f\u53ea\u88ab\u6d88\u8d39\u4e00\u6b21\u7684\u8bed\u4e49\uff0cKafka\u4fdd\u8bc1\u4fdd\u8bc1\u540c\u4e00\u4e2aconsumer group\u91cc\u53ea\u6709\u4e00\u4e2aconsumer\u4f1a\u6d88\u8d39\u4e00\u6761\u6d88\u606f\u3002\u4e0e\u4f20\u7edfmessage queue\u4e0d\u540c\u7684\u662f\uff0cKafka\u8fd8\u5141\u8bb8\u4e0d\u540cconsumer group\u540c\u65f6\u6d88\u8d39\u540c\u4e00\u6761\u6d88\u606f\uff0c\u8fd9\u4e00\u7279\u6027\u53ef\u4ee5\u4e3a\u6d88\u606f\u7684\u591a\u5143\u5316\u5904\u7406\u63d0\u4f9b\u4e86\u652f\u6301\u3002\u5b9e\u9645\u4e0a\uff0cKafka\u7684\u8bbe\u8ba1\u7406\u5ff5\u4e4b\u4e00\u5c31\u662f\u540c\u65f6\u63d0\u4f9b\u79bb\u7ebf\u5904\u7406\u548c\u5b9e\u65f6\u5904\u7406\u3002\u6839\u636e\u8fd9\u4e00\u7279\u6027\uff0c\u53ef\u4ee5\u4f7f\u7528Storm\u8fd9\u79cd\u5b9e\u65f6\u6d41\u5904\u7406\u7cfb\u7edf\u5bf9\u6d88\u606f\u8fdb\u884c\u5b9e\u65f6\u5728\u7ebf\u5904\u7406\uff0c\u540c\u65f6\u4f7f\u7528Hadoop\u8fd9\u79cd\u6279\u5904\u7406\u7cfb\u7edf\u8fdb\u884c\u79bb\u7ebf\u5904\u7406\uff0c\u8fd8\u53ef\u4ee5\u540c\u65f6\u5c06\u6570\u636e\u5b9e\u65f6\u5907\u4efd\u5230\u53e6\u4e00\u4e2a\u6570\u636e\u4e2d\u5fc3\uff0c\u53ea\u9700\u8981\u4fdd\u8bc1\u8fd9\u4e09\u4e2a\u64cd\u4f5c\u6240\u4f7f\u7528\u7684consumer\u5728\u4e0d\u540c\u7684consumer group\u5373\u53ef\u3002\u4e0b\u56fe\u5c55\u793a\u4e86Kafka\u5728Linkedin\u7684\u4e00\u79cd\u7b80\u5316\u90e8\u7f72\u3002<\/p>\n<\/div>\n<div><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1502\" rel=\"attachment wp-att-1502\"><img loading=\"lazy\" class=\"alignnone size-full wp-image-1502\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/9.png\" alt=\"\" width=\"686\" height=\"631\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/9.png 686w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/9-300x276.png 300w\" sizes=\"(max-width: 686px) 100vw, 686px\" \/><\/a><\/div>\n<div><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1503\" rel=\"attachment wp-att-1503\"><img loading=\"lazy\" class=\"alignnone size-large wp-image-1503\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/10-1024x637.png\" alt=\"\" width=\"660\" height=\"411\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/10-1024x637.png 1024w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/10-300x187.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/10-768x478.png 768w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/10.png 1455w\" sizes=\"(max-width: 660px) 100vw, 660px\" \/><\/a><\/div>\n<div>\u4e3a\u4e86\u66f4\u6e05\u6670\u5c55\u793aKafka consumer group\u7684\u7279\u6027\uff0c\u7b14\u8005\u4f5c\u4e86\u4e00\u9879\u6d4b\u8bd5\u3002\u521b\u5efa\u4e00\u4e2atopic (\u540d\u4e3atopic1)\uff0c\u521b\u5efa\u4e00\u4e2a\u5c5e\u4e8egroup1\u7684consumer\u5b9e\u4f8b\uff0c\u5e76\u521b\u5efa\u4e09\u4e2a\u5c5e\u4e8egroup2\u7684consumer\u5b9e\u4f8b\uff0c\u7136\u540e\u901a\u8fc7producer\u5411topic1\u53d1\u9001key\u5206\u522b\u4e3a1\uff0c2\uff0c3r\u7684\u6d88\u606f\u3002\u7ed3\u679c\u53d1\u73b0\u5c5e\u4e8egroup1\u7684consumer\u6536\u5230\u4e86\u6240\u6709\u7684\u8fd9\u4e09\u6761\u6d88\u606f\uff0c\u540c\u65f6group2\u4e2d\u76843\u4e2aconsumer\u5206\u522b\u6536\u5230\u4e86key\u4e3a1\uff0c2\uff0c3\u7684\u6d88\u606f\u3002\u5982\u4e0b\u56fe\u6240\u793a\u3002<\/div>\n<h3>Consumer Rebalance<\/h3>\n<div>\n<p>\uff08\u672c\u8282\u6240\u8bb2\u8ff0\u5185\u5bb9\u5747\u57fa\u4e8eKafka consumer high level API\uff09<\/p>\n<p>Kafka\u4fdd\u8bc1\u540c\u4e00consumer group\u4e2d\u53ea\u6709\u4e00\u4e2aconsumer\u4f1a\u6d88\u606f\u67d0\u6761\u6d88\u606f\uff0c\u5b9e\u9645\u4e0a\uff0cKafka\u4fdd\u8bc1\u7684\u662f\u7a33\u5b9a\u72b6\u6001\u4e0b\u6bcf\u4e00\u4e2aconsumer\u5b9e\u4f8b\u53ea\u4f1a\u6d88\u8d39\u67d0\u4e00\u4e2a\u6216\u591a\u4e2a\u7279\u5b9apartition\u7684\u6570\u636e\uff0c\u800c\u67d0\u4e2apartition\u7684\u6570\u636e\u53ea\u4f1a\u88ab\u67d0\u4e00\u4e2a\u7279\u5b9a\u7684consumer\u5b9e\u4f8b\u6240\u6d88\u8d39\u3002\u8fd9\u6837\u8bbe\u8ba1\u7684\u52a3\u52bf\u662f\u65e0\u6cd5\u8ba9\u540c\u4e00\u4e2aconsumer group\u91cc\u7684consumer\u5747\u5300\u6d88\u8d39\u6570\u636e\uff0c\u4f18\u52bf\u662f\u6bcf\u4e2aconsumer\u4e0d\u7528\u90fd\u8ddf\u5927\u91cf\u7684broker\u901a\u4fe1\uff0c\u51cf\u5c11\u901a\u4fe1\u5f00\u9500\uff0c\u540c\u65f6\u4e5f\u964d\u4f4e\u4e86\u5206\u914d\u96be\u5ea6\uff0c\u5b9e\u73b0\u4e5f\u66f4\u7b80\u5355\u3002\u53e6\u5916\uff0c\u56e0\u4e3a\u540c\u4e00\u4e2apartition\u91cc\u7684\u6570\u636e\u662f\u6709\u5e8f\u7684\uff0c\u8fd9\u79cd\u8bbe\u8ba1\u53ef\u4ee5\u4fdd\u8bc1\u6bcf\u4e2apartition\u91cc\u7684\u6570\u636e\u4e5f\u662f\u6709\u5e8f\u88ab\u6d88\u8d39\u3002<\/p>\n<p>\u5982\u679c\u67d0consumer group\u4e2dconsumer\u6570\u91cf\u5c11\u4e8epartition\u6570\u91cf\uff0c\u5219\u81f3\u5c11\u6709\u4e00\u4e2aconsumer\u4f1a\u6d88\u8d39\u591a\u4e2apartition\u7684\u6570\u636e\uff0c\u5982\u679cconsumer\u7684\u6570\u91cf\u4e0epartition\u6570\u91cf\u76f8\u540c\uff0c\u5219\u6b63\u597d\u4e00\u4e2aconsumer\u6d88\u8d39\u4e00\u4e2apartition\u7684\u6570\u636e\uff0c\u800c\u5982\u679cconsumer\u7684\u6570\u91cf\u591a\u4e8epartition\u7684\u6570\u91cf\u65f6\uff0c\u4f1a\u6709\u90e8\u5206consumer\u65e0\u6cd5\u6d88\u8d39\u8be5topic\u4e0b\u4efb\u4f55\u4e00\u6761\u6d88\u606f\u3002<\/p>\n<p>\u5982\u4e0b\u4f8b\u6240\u793a\uff0c\u5982\u679ctopic1\u67090\uff0c1\uff0c2\u5171\u4e09\u4e2apartition\uff0c\u5f53group1\u53ea\u6709\u4e00\u4e2aconsumer(\u540d\u4e3aconsumer1)\u65f6\uff0c\u8be5 consumer\u53ef\u6d88\u8d39\u8fd93\u4e2apartition\u7684\u6240\u6709\u6570\u636e\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1504\" rel=\"attachment wp-att-1504\"><img loading=\"lazy\" class=\"alignnone size-large wp-image-1504\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/11-1024x166.png\" alt=\"\" width=\"660\" height=\"107\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/11-1024x166.png 1024w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/11-300x49.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/11-768x124.png 768w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/11.png 1401w\" sizes=\"(max-width: 660px) 100vw, 660px\" \/><\/a><br \/>\n\u589e\u52a0\u4e00\u4e2aconsumer(consumer2)\u540e\uff0c\u5176\u4e2d\u4e00\u4e2aconsumer\uff08consumer1\uff09\u53ef\u6d88\u8d392\u4e2apartition\u7684\u6570\u636e\uff0c\u53e6\u5916\u4e00\u4e2aconsumer(consumer2)\u53ef\u6d88\u8d39\u53e6\u5916\u4e00\u4e2apartition\u7684\u6570\u636e\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1505\" rel=\"attachment wp-att-1505\"><img loading=\"lazy\" class=\"alignnone size-large wp-image-1505\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/12-1024x364.png\" alt=\"\" width=\"660\" height=\"235\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/12-1024x364.png 1024w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/12-300x107.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/12-768x273.png 768w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/12.png 1429w\" sizes=\"(max-width: 660px) 100vw, 660px\" \/><\/a><br \/>\n\u518d\u589e\u52a0\u4e00\u4e2aconsumer(consumer3)\u540e\uff0c\u6bcf\u4e2aconsumer\u53ef\u6d88\u8d39\u4e00\u4e2apartition\u7684\u6570\u636e\u3002consumer1\u6d88\u8d39partition0\uff0cconsumer2\u6d88\u8d39partition1\uff0cconsumer3\u6d88\u8d39partition2<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1506\" rel=\"attachment wp-att-1506\"><img loading=\"lazy\" class=\"alignnone size-large wp-image-1506\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/13-1024x480.png\" alt=\"\" width=\"660\" height=\"309\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/13-1024x480.png 1024w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/13-300x141.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/13-768x360.png 768w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/13.png 1416w\" sizes=\"(max-width: 660px) 100vw, 660px\" \/><\/a><br \/>\n\u518d\u589e\u52a0\u4e00\u4e2aconsumer\uff08consumer4\uff09\u540e\uff0c\u5176\u4e2d3\u4e2aconsumer\u53ef\u5206\u522b\u6d88\u8d39\u4e00\u4e2apartition\u7684\u6570\u636e\uff0c\u53e6\u5916\u4e00\u4e2aconsumer\uff08consumer4\uff09\u4e0d\u80fd\u6d88\u8d39topic1\u4efb\u4f55\u6570\u636e\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1507\" rel=\"attachment wp-att-1507\"><img loading=\"lazy\" class=\"alignnone size-large wp-image-1507\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/14-1024x531.png\" alt=\"\" width=\"660\" height=\"342\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/14-1024x531.png 1024w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/14-300x156.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/14-768x398.png 768w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/14.png 1429w\" sizes=\"(max-width: 660px) 100vw, 660px\" \/><\/a><br \/>\n\u6b64\u65f6\u5173\u95edconsumer1\uff0c\u5269\u4e0b\u7684consumer\u53ef\u5206\u522b\u6d88\u8d39\u4e00\u4e2apartition\u7684\u6570\u636e\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1508\" rel=\"attachment wp-att-1508\"><img loading=\"lazy\" class=\"alignnone size-large wp-image-1508\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/15-1024x484.png\" alt=\"\" width=\"660\" height=\"312\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/15-1024x484.png 1024w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/15-300x142.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/15-768x363.png 768w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/15.png 1417w\" sizes=\"(max-width: 660px) 100vw, 660px\" \/><\/a><br \/>\n\u63a5\u7740\u5173\u95edconsumer2\uff0c\u5269\u4e0b\u7684consumer3\u53ef\u6d88\u8d392\u4e2apartition\uff0cconsumer4\u53ef\u6d88\u8d391\u4e2apartition\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1509\" rel=\"attachment wp-att-1509\"><img loading=\"lazy\" class=\"alignnone size-large wp-image-1509\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/16-1024x354.png\" alt=\"\" width=\"660\" height=\"228\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/16-1024x354.png 1024w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/16-300x104.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/16-768x266.png 768w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/16.png 1413w\" sizes=\"(max-width: 660px) 100vw, 660px\" \/><\/a><br \/>\n\u518d\u5173\u95edconsumer3\uff0c\u5269\u4e0b\u7684consumer4\u53ef\u540c\u65f6\u6d88\u8d39topic1\u76843\u4e2apartition\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1510\" rel=\"attachment wp-att-1510\"><img loading=\"lazy\" class=\"alignnone size-large wp-image-1510\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/17-1024x208.png\" alt=\"\" width=\"660\" height=\"134\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/17-1024x208.png 1024w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/17-300x61.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/17-768x156.png 768w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/17.png 1408w\" sizes=\"(max-width: 660px) 100vw, 660px\" \/><\/a><\/p>\n<\/div>\n<p>consumer rebalance\u7b97\u6cd5\u5982\u4e0b\uff1a<\/p>\n<ul>\n<li>Sort PT (all partitions in topic T)<\/li>\n<li>Sort CG(all consumers in consumer group G)<\/li>\n<li>Let i be the index position of Ci in CG and let N=size(PT)\/size(CG)<\/li>\n<li>Remove current entries owned by Ci from the partition owner registry<\/li>\n<li>Assign partitions from i\u00a0<em>N to (i+1)<\/em>\u00a0N-1 to consumer Ci<\/li>\n<li>Add newly assigned partitions to the partition owner registry<\/li>\n<\/ul>\n<p>\u76ee\u524dconsumer rebalance\u7684\u63a7\u5236\u7b56\u7565\u662f\u7531\u6bcf\u4e00\u4e2aconsumer\u901a\u8fc7Zookeeper\u5b8c\u6210\u7684\u3002\u5177\u4f53\u7684\u63a7\u5236\u65b9\u5f0f\u5982\u4e0b\uff1a<\/p>\n<ul>\n<li>Register itself in the consumer id registry under its group.<\/li>\n<li>Register a watch on changes under the consumer id registry.<\/li>\n<li>Register a watch on changes under the broker id registry.<\/li>\n<li>If the consumer creates a message stream using a topic filter, it also registers a watch on changes under the broker topic registry.<\/li>\n<li>Force itself to rebalance within in its consumer group.\u5728\u8fd9\u79cd\u7b56\u7565\u4e0b\uff0c\u6bcf\u4e00\u4e2aconsumer\u6216\u8005broker\u7684\u589e\u52a0\u6216\u8005\u51cf\u5c11\u90fd\u4f1a\u89e6\u53d1consumer rebalance\u3002\u56e0\u4e3a\u6bcf\u4e2aconsumer\u53ea\u8d1f\u8d23\u8c03\u6574\u81ea\u5df1\u6240\u6d88\u8d39\u7684partition\uff0c\u4e3a\u4e86\u4fdd\u8bc1\u6574\u4e2aconsumer group\u7684\u4e00\u81f4\u6027\uff0c\u6240\u4ee5\u5f53\u4e00\u4e2aconsumer\u89e6\u53d1\u4e86rebalance\u65f6\uff0c\u8be5consumer group\u5185\u7684\u5176\u5b83\u6240\u6709consumer\u4e5f\u5e94\u8be5\u540c\u65f6\u89e6\u53d1rebalance\u3002<\/li>\n<\/ul>\n<p>\u76ee\u524d\uff082015-01-19\uff09\u6700\u65b0\u7248\uff080.8.2\uff09Kafka\u91c7\u7528\u7684\u662f\u4e0a\u8ff0\u65b9\u5f0f\u3002\u4f46\u8be5\u65b9\u5f0f\u6709\u4e0d\u5229\u7684\u65b9\u9762\uff1a<\/p>\n<ul>\n<li><b>Herd effect<\/b><br \/>\n\u4efb\u4f55broker\u6216\u8005consumer\u7684\u589e\u51cf\u90fd\u4f1a\u89e6\u53d1\u6240\u6709\u7684consumer\u7684rebalance<\/li>\n<li><b>Split Brain<\/b><br \/>\n\u6bcf\u4e2aconsumer\u5206\u522b\u5355\u72ec\u901a\u8fc7Zookeeper\u5224\u65ad\u54ea\u4e9bpartition down\u4e86\uff0c\u90a3\u4e48\u4e0d\u540cconsumer\u4eceZookeeper\u201c\u770b\u201d\u5230\u7684view\u5c31\u53ef\u80fd\u4e0d\u4e00\u6837\uff0c\u8fd9\u5c31\u4f1a\u9020\u6210\u9519\u8bef\u7684reblance\u5c1d\u8bd5\u3002\u800c\u4e14\u6709\u53ef\u80fd\u6240\u6709\u7684consumer\u90fd\u8ba4\u4e3arebalance\u5df2\u7ecf\u5b8c\u6210\u4e86\uff0c\u4f46\u5b9e\u9645\u4e0a\u53ef\u80fd\u5e76\u975e\u5982\u6b64\u3002<\/li>\n<\/ul>\n<div>\n<p>\u6839\u636eKafka\u5b98\u65b9\u6587\u6863\uff0cKafka\u4f5c\u8005\u6b63\u5728\u8003\u8651\u5728\u8fd8\u672a\u53d1\u5e03\u7684\u00a0<a href=\"https:\/\/cwiki.apache.org\/confluence\/display\/KAFKA\/Kafka+0.9+Consumer+Rewrite+Design\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">0.9.x\u7248\u672c\u4e2d\u4f7f\u7528\u4e2d\u5fc3\u534f\u8c03\u5668(coordinator)<\/a>\u00a0\u3002\u5927\u4f53\u601d\u60f3\u662f\u9009\u4e3e\u51fa\u4e00\u4e2abroker\u4f5c\u4e3acoordinator\uff0c\u7531\u5b83watch Zookeeper\uff0c\u4ece\u800c\u5224\u65ad\u662f\u5426\u6709partition\u6216\u8005consumer\u7684\u589e\u51cf\uff0c\u7136\u540e\u751f\u6210rebalance\u547d\u4ee4\uff0c\u5e76\u68c0\u67e5\u662f\u5426\u8fd9\u4e9brebalance\u5728\u6240\u6709\u76f8\u5173\u7684consumer\u4e2d\u88ab\u6267\u884c\u6210\u529f\uff0c\u5982\u679c\u4e0d\u6210\u529f\u5219\u91cd\u8bd5\uff0c\u82e5\u6210\u529f\u5219\u8ba4\u4e3a\u6b64\u6b21rebalance\u6210\u529f\uff08\u8fd9\u4e2a\u8fc7\u7a0b\u8ddfreplication controller\u975e\u5e38\u7c7b\u4f3c\uff0c\u6240\u4ee5\u6211\u5f88\u5947\u602a\u4e3a\u4ec0\u4e48\u5f53\u521d\u8bbe\u8ba1replication controller\u65f6\u6ca1\u6709\u4f7f\u7528\u7c7b\u4f3c\u65b9\u5f0f\u6765\u89e3\u51b3consumer rebalance\u7684\u95ee\u9898\uff09\u3002\u6d41\u7a0b\u5982\u4e0b\uff1a<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1511\" rel=\"attachment wp-att-1511\"><img loading=\"lazy\" class=\"alignnone size-large wp-image-1511\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/18-1024x673.png\" alt=\"\" width=\"660\" height=\"434\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/18-1024x673.png 1024w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/18-300x197.png 300w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/18-768x504.png 768w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/18.png 1125w\" sizes=\"(max-width: 660px) 100vw, 660px\" \/><\/a><\/p>\n<\/div>\n<h3>\u6d88\u606fDeliver guarantee<\/h3>\n<p>\u901a\u8fc7\u4e0a\u6587\u4ecb\u7ecd\uff0c\u60f3\u5fc5\u8bfb\u8005\u5df2\u7ecf\u660e\u5929\u4e86producer\u548cconsumer\u662f\u5982\u4f55\u5de5\u4f5c\u7684\uff0c\u4ee5\u53caKafka\u662f\u5982\u4f55\u505areplication\u7684\uff0c\u63a5\u4e0b\u6765\u8981\u8ba8\u8bba\u7684\u662fKafka\u5982\u4f55\u786e\u4fdd\u6d88\u606f\u5728producer\u548cconsumer\u4e4b\u95f4\u4f20\u8f93\u3002\u6709\u8fd9\u4e48\u51e0\u79cd\u53ef\u80fd\u7684delivery guarantee\uff1a<\/p>\n<ul>\n<li><code>At most once<\/code>\u00a0\u6d88\u606f\u53ef\u80fd\u4f1a\u4e22\uff0c\u4f46\u7edd\u4e0d\u4f1a\u91cd\u590d\u4f20\u8f93<\/li>\n<li><code>At least one<\/code>\u00a0\u6d88\u606f\u7edd\u4e0d\u4f1a\u4e22\uff0c\u4f46\u53ef\u80fd\u4f1a\u91cd\u590d\u4f20\u8f93<\/li>\n<li><code>Exactly once<\/code>\u00a0\u6bcf\u6761\u6d88\u606f\u80af\u5b9a\u4f1a\u88ab\u4f20\u8f93\u4e00\u6b21\u4e14\u4ec5\u4f20\u8f93\u4e00\u6b21\uff0c\u5f88\u591a\u65f6\u5019\u8fd9\u662f\u7528\u6237\u6240\u60f3\u8981\u7684\u3002\n<div>\n<p>Kafka\u7684delivery guarantee semantic\u975e\u5e38\u76f4\u63a5\u3002\u5f53producer\u5411broker\u53d1\u9001\u6d88\u606f\u65f6\uff0c\u4e00\u65e6\u8fd9\u6761\u6d88\u606f\u88abcommit\uff0c\u56e0\u6570replication\u7684\u5b58\u5728\uff0c\u5b83\u5c31\u4e0d\u4f1a\u4e22\u3002\u4f46\u662f\u5982\u679cproducer\u53d1\u9001\u6570\u636e\u7ed9broker\u540e\uff0c\u9047\u5230\u7684\u7f51\u7edc\u95ee\u9898\u800c\u9020\u6210\u901a\u4fe1\u4e2d\u65ad\uff0c\u90a3producer\u5c31\u65e0\u6cd5\u5224\u65ad\u8be5\u6761\u6d88\u606f\u662f\u5426\u5df2\u7ecfcommit\u3002\u8fd9\u4e00\u70b9\u6709\u70b9\u50cf\u5411\u4e00\u4e2a\u81ea\u52a8\u751f\u6210primary key\u7684\u6570\u636e\u5e93\u8868\u4e2d\u63d2\u5165\u6570\u636e\u3002\u867d\u7136Kafka\u65e0\u6cd5\u786e\u5b9a\u7f51\u7edc\u6545\u969c\u671f\u95f4\u53d1\u751f\u4e86\u4ec0\u4e48\uff0c\u4f46\u662fproducer\u53ef\u4ee5\u751f\u6210\u4e00\u79cd\u7c7b\u4f3c\u4e8eprimary key\u7684\u4e1c\u897f\uff0c\u53d1\u751f\u6545\u969c\u65f6\u5e42\u7b49\u6027\u7684retry\u591a\u6b21\uff0c\u8fd9\u6837\u5c31\u505a\u5230\u4e86\u00a0<code>Exactly one<\/code>\u00a0\u3002\u622a\u6b62\u5230\u76ee\u524d(Kafka 0.8.2\u7248\u672c\uff0c2015-01-25)\uff0c\u8fd9\u4e00feature\u8fd8\u5e76\u672a\u5b9e\u73b0\uff0c\u6709\u5e0c\u671b\u5728Kafka\u672a\u6765\u7684\u7248\u672c\u4e2d\u5b9e\u73b0\u3002\uff08\u6240\u4ee5\u76ee\u524d\u9ed8\u8ba4\u60c5\u51b5\u4e0b\u4e00\u6761\u6d88\u606f\u4eceproducer\u548cbroker\u662f\u786e\u4fdd\u4e86\u00a0<code>At least once<\/code>\u00a0\uff0c\u4f46\u53ef\u901a\u8fc7\u8bbe\u7f6eproducer\u5f02\u6b65\u53d1\u9001\u5b9e\u73b0\u00a0<code>At most once<\/code>\uff09\u3002<\/p>\n<p>\u63a5\u4e0b\u6765\u8ba8\u8bba\u7684\u662f\u6d88\u606f\u4ecebroker\u5230consumer\u7684delivery guarantee semantic\u3002\uff08\u4ec5\u9488\u5bf9Kafka consumer high level API\uff09\u3002consumer\u5728\u4ecebroker\u8bfb\u53d6\u6d88\u606f\u540e\uff0c\u53ef\u4ee5\u9009\u62e9commit\uff0c\u8be5\u64cd\u4f5c\u4f1a\u5728Zookeeper\u4e2d\u5b58\u4e0b\u8be5consumer\u5728\u8be5partition\u4e0b\u8bfb\u53d6\u7684\u6d88\u606f\u7684offset\u3002\u8be5consumer\u4e0b\u4e00\u6b21\u518d\u8bfb\u8be5partition\u65f6\u4f1a\u4ece\u4e0b\u4e00\u6761\u5f00\u59cb\u8bfb\u53d6\u3002\u5982\u672acommit\uff0c\u4e0b\u4e00\u6b21\u8bfb\u53d6\u7684\u5f00\u59cb\u4f4d\u7f6e\u4f1a\u8ddf\u4e0a\u4e00\u6b21commit\u4e4b\u540e\u7684\u5f00\u59cb\u4f4d\u7f6e\u76f8\u540c\u3002\u5f53\u7136\u53ef\u4ee5\u5c06consumer\u8bbe\u7f6e\u4e3aautocommit\uff0c\u5373consumer\u4e00\u65e6\u8bfb\u5230\u6570\u636e\u7acb\u5373\u81ea\u52a8commit\u3002\u5982\u679c\u53ea\u8ba8\u8bba\u8fd9\u4e00\u8bfb\u53d6\u6d88\u606f\u7684\u8fc7\u7a0b\uff0c\u90a3Kafka\u662f\u786e\u4fdd\u4e86\u00a0<code>Exactly once<\/code>\u00a0\u3002\u4f46\u5b9e\u9645\u4e0a\u5b9e\u9645\u4f7f\u7528\u4e2dconsumer\u5e76\u975e\u8bfb\u53d6\u5b8c\u6570\u636e\u5c31\u7ed3\u675f\u4e86\uff0c\u800c\u662f\u8981\u8fdb\u884c\u8fdb\u4e00\u6b65\u5904\u7406\uff0c\u800c\u6570\u636e\u5904\u7406\u4e0ecommit\u7684\u987a\u5e8f\u5728\u5f88\u5927\u7a0b\u5ea6\u4e0a\u51b3\u5b9a\u4e86\u6d88\u606f\u4ecebroker\u548cconsumer\u7684delivery guarantee semantic\u3002<\/p>\n<\/div>\n<\/li>\n<li>\u8bfb\u5b8c\u6d88\u606f\u5148commit\u518d\u5904\u7406\u6d88\u606f\u3002\u8fd9\u79cd\u6a21\u5f0f\u4e0b\uff0c\u5982\u679cconsumer\u5728commit\u540e\u8fd8\u6ca1\u6765\u5f97\u53ca\u5904\u7406\u6d88\u606f\u5c31crash\u4e86\uff0c\u4e0b\u6b21\u91cd\u65b0\u5f00\u59cb\u5de5\u4f5c\u540e\u5c31\u65e0\u6cd5\u8bfb\u5230\u521a\u521a\u5df2\u63d0\u4ea4\u800c\u672a\u5904\u7406\u7684\u6d88\u606f\uff0c\u8fd9\u5c31\u5bf9\u5e94\u4e8e\u00a0<code>At most once<\/code><\/li>\n<li>\u8bfb\u5b8c\u6d88\u606f\u5148\u5904\u7406\u518dcommit\u3002\u8fd9\u79cd\u6a21\u5f0f\u4e0b\uff0c\u5982\u679c\u5904\u7406\u5b8c\u4e86\u6d88\u606f\u5728commit\u4e4b\u524dconsumer crash\u4e86\uff0c\u4e0b\u6b21\u91cd\u65b0\u5f00\u59cb\u5de5\u4f5c\u65f6\u8fd8\u4f1a\u5904\u7406\u521a\u521a\u672acommit\u7684\u6d88\u606f\uff0c\u5b9e\u9645\u4e0a\u8be5\u6d88\u606f\u5df2\u7ecf\u88ab\u5904\u7406\u8fc7\u4e86\u3002\u8fd9\u5c31\u5bf9\u5e94\u4e8e\u00a0<code>At least once<\/code>\u00a0\u3002\u5728\u5f88\u591a\u60c5\u51b5\u4f7f\u7528\u573a\u666f\u4e0b\uff0c\u6d88\u606f\u90fd\u6709\u4e00\u4e2aprimary key\uff0c\u6240\u4ee5\u6d88\u606f\u7684\u5904\u7406\u5f80\u5f80\u5177\u6709\u5e42\u7b49\u6027\uff0c\u5373\u591a\u6b21\u5904\u7406\u8fd9\u4e00\u6761\u6d88\u606f\u8ddf\u53ea\u5904\u7406\u4e00\u6b21\u662f\u7b49\u6548\u7684\uff0c\u90a3\u5c31\u53ef\u4ee5\u8ba4\u4e3a\u662f\u00a0<code>Exactly once<\/code>\u00a0\u3002\uff08\u4eba\u4e2a\u611f\u89c9\u8fd9\u79cd\u8bf4\u6cd5\u6709\u4e9b\u7275\u5f3a\uff0c\u6bd5\u7adf\u5b83\u4e0d\u662fKafka\u672c\u8eab\u63d0\u4f9b\u7684\u673a\u5236\uff0c\u800c\u4e14primary key\u672c\u8eab\u4e0d\u4fdd\u8bc1\u64cd\u4f5c\u7684\u5e42\u7b49\u6027\u3002\u800c\u4e14\u5b9e\u9645\u4e0a\u6211\u4eec\u8bf4delivery guarantee semantic\u662f\u8ba8\u8bba\u88ab\u5904\u7406\u591a\u5c11\u6b21\uff0c\u800c\u975e\u5904\u7406\u7ed3\u679c\u600e\u6837\uff0c\u56e0\u4e3a\u5904\u7406\u65b9\u5f0f\u591a\u79cd\u591a\u6837\uff0c\u6211\u4eec\u7684\u7cfb\u7edf\u4e0d\u5e94\u8be5\u628a\u5904\u7406\u8fc7\u7a0b\u7684\u7279\u6027\u2014\u5982\u662f\u5426\u5e42\u7b49\u6027\uff0c\u5f53\u6210Kafka\u672c\u8eab\u7684feature\uff09<\/li>\n<li>\u5982\u679c\u4e00\u5b9a\u8981\u505a\u5230\u00a0<code>Exactly once<\/code>\u00a0\uff0c\u5c31\u9700\u8981\u534f\u8c03offset\u548c\u5b9e\u9645\u64cd\u4f5c\u7684\u8f93\u51fa\u3002\u7cbe\u5178\u7684\u505a\u6cd5\u662f\u5f15\u5165\u4e24\u9636\u6bb5\u63d0\u4ea4\u3002\u5982\u679c\u80fd\u8ba9offset\u548c\u64cd\u4f5c\u8f93\u5165\u5b58\u5728\u540c\u4e00\u4e2a\u5730\u65b9\uff0c\u4f1a\u66f4\u7b80\u6d01\u548c\u901a\u7528\u3002\u8fd9\u79cd\u65b9\u5f0f\u53ef\u80fd\u66f4\u597d\uff0c\u56e0\u4e3a\u8bb8\u591a\u8f93\u51fa\u7cfb\u7edf\u53ef\u80fd\u4e0d\u652f\u6301\u4e24\u9636\u6bb5\u63d0\u4ea4\u3002\u6bd4\u5982\uff0cconsumer\u62ff\u5230\u6570\u636e\u540e\u53ef\u80fd\u628a\u6570\u636e\u653e\u5230HDFS\uff0c\u5982\u679c\u628a\u6700\u65b0\u7684offset\u548c\u6570\u636e\u672c\u8eab\u4e00\u8d77\u5199\u5230HDFS\uff0c\u90a3\u5c31\u53ef\u4ee5\u4fdd\u8bc1\u6570\u636e\u7684\u8f93\u51fa\u548coffset\u7684\u66f4\u65b0\u8981\u4e48\u90fd\u5b8c\u6210\uff0c\u8981\u4e48\u90fd\u4e0d\u5b8c\u6210\uff0c\u95f4\u63a5\u5b9e\u73b0\u00a0<code>Exactly once<\/code>\u00a0\u3002\uff08\u76ee\u524d\u5c31high level API\u800c\u8a00\uff0coffset\u662f\u5b58\u4e8eZookeeper\u4e2d\u7684\uff0c\u65e0\u6cd5\u5b58\u4e8eHDFS\uff0c\u800clow level API\u7684offset\u662f\u7531\u81ea\u5df1\u53bb\u7ef4\u62a4\u7684\uff0c\u53ef\u4ee5\u5c06\u4e4b\u5b58\u4e8eHDFS\u4e2d\uff09<br \/>\n\u603b\u4e4b\uff0cKafka\u9ed8\u8ba4\u4fdd\u8bc1\u00a0<code>At least once<\/code>\u00a0\uff0c\u5e76\u4e14\u5141\u8bb8\u901a\u8fc7\u8bbe\u7f6eproducer\u5f02\u6b65\u63d0\u4ea4\u6765\u5b9e\u73b0\u00a0<code>At most once<\/code>\u00a0\u3002\u800c\u00a0<code>Exactly once<\/code>\u00a0\u8981\u6c42\u4e0e\u76ee\u6807\u5b58\u50a8\u7cfb\u7edf\u534f\u4f5c\uff0c\u5e78\u8fd0\u7684\u662fKafka\u63d0\u4f9b\u7684offset\u53ef\u4ee5\u4f7f\u7528\u8fd9\u79cd\u65b9\u5f0f\u975e\u5e38\u76f4\u63a5\u975e\u5e38\u5bb9\u6613\u3002<\/li>\n<\/ul>\n<h2>Benchmark<\/h2>\n<p>\u7eb8\u4e0a\u5f97\u6765\u7ec8\u89c9\u6d45\uff0c\u7edd\u77e5\u4e9b\u4e8b\u8981\u8eac\u884c\u3002\u7b14\u8005\u5e0c\u671b\u80fd\u4eb2\u81ea\u6d4b\u4e00\u4e0bKafka\u7684\u6027\u80fd\uff0c\u800c\u975e\u4ece\u7f51\u4e0a\u627e\u4e00\u4e9b\u6d4b\u8bd5\u6570\u636e\u3002\u6240\u4ee5\u7b14\u8005\u66fe\u57280.8\u53d1\u5e03\u524d\u4e24\u4e2a\u6708\u505a\u8fc7\u8be6\u7ec6\u7684Kafka0.8\u6027\u80fd\u6d4b\u8bd5\uff0c\u4e0d\u8fc7\u5f88\u53ef\u60dc\u6d4b\u8bd5\u62a5\u544a\u4e0d\u614e\u4e22\u5931\u3002\u6240\u5e78\u5728\u7f51\u4e0a\u627e\u5230\u4e86Kafka\u7684\u521b\u59cb\u4eba\u4e4b\u4e00\u7684\u00a0<a href=\"http:\/\/engineering.linkedin.com\/kafka\/benchmarking-apache-kafka-2-million-writes-second-three-cheap-machines\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">Jay Kreps\u7684bechmark<\/a>\u00a0\u3002\u4ee5\u4e0b\u63cf\u8ff0\u7686\u57fa\u4e8e\u8be5benchmark\u3002\uff08\u8be5benchmark\u57fa\u4e8eKafka0.8.1\uff09<\/p>\n<h2>\u6d4b\u8bd5\u73af\u5883<\/h2>\n<p>\u8be5benchmark\u7528\u5230\u4e86\u516d\u53f0\u673a\u5668\uff0c\u673a\u5668\u914d\u7f6e\u5982\u4e0b<\/p>\n<ul>\n<li>Intel Xeon 2.5 GHz processor with six cores<\/li>\n<li>Six 7200 RPM SATA drives<\/li>\n<li>32GB of RAM<\/li>\n<li>1Gb Ethernet\u8fd96\u53f0\u673a\u5668\u5176\u4e2d3\u53f0\u7528\u6765\u642d\u5efaKafka broker\u96c6\u7fa4\uff0c\u53e6\u59163\u53f0\u7528\u6765\u5b89\u88c5Zookeeper\u53ca\u751f\u6210\u6d4b\u8bd5\u6570\u636e\u30026\u4e2adrive\u90fd\u76f4\u63a5\u4ee5\u975eRAID\u65b9\u5f0f\u6302\u8f7d\u3002\u5b9e\u9645\u4e0akafka\u5bf9\u673a\u5668\u7684\u9700\u6c42\u4e0eHadoop\u7684\u7c7b\u4f3c\u3002<\/li>\n<\/ul>\n<h2>producer\u541e\u5410\u7387<\/h2>\n<p>\u8be5\u9879\u6d4b\u8bd5\u53ea\u6d4bproducer\u7684\u541e\u5410\u7387\uff0c\u4e5f\u5c31\u662f\u6570\u636e\u53ea\u88ab\u6301\u4e45\u5316\uff0c\u6ca1\u6709consumer\u8bfb\u6570\u636e\u3002<\/p>\n<h3>1\u4e2aproducer\u7ebf\u7a0b\uff0c\u65e0replication<\/h3>\n<div>\n<p>\u5728\u8fd9\u4e00\u6d4b\u8bd5\u4e2d\uff0c\u521b\u5efa\u4e86\u4e00\u4e2a\u5305\u542b6\u4e2apartition\u4e14\u6ca1\u6709replication\u7684topic\u3002\u7136\u540e\u901a\u8fc7\u4e00\u4e2a\u7ebf\u7a0b\u5c3d\u53ef\u80fd\u5feb\u7684\u751f\u621050 million\u6761\u6bd4\u8f83\u77ed\uff08payload100\u5b57\u8282\u957f\uff09\u7684\u6d88\u606f\u3002\u6d4b\u8bd5\u7ed3\u679c\u662f\u00a0<strong><em>821,557 records\/second<\/em>\u00a0<\/strong>\uff08\u00a0<strong><em>78.3MB\/second<\/em>\u00a0<\/strong>\uff09\u3002<\/p>\n<p>\u4e4b\u6240\u4ee5\u4f7f\u7528\u77ed\u6d88\u606f\uff0c\u662f\u56e0\u4e3a\u5bf9\u4e8e\u6d88\u606f\u7cfb\u7edf\u6765\u8bf4\u8fd9\u79cd\u4f7f\u7528\u573a\u666f\u66f4\u96be\u3002\u56e0\u4e3a\u5982\u679c\u4f7f\u7528MB\/second\u6765\u8868\u5f81\u541e\u5410\u7387\uff0c\u90a3\u53d1\u9001\u957f\u6d88\u606f\u65e0\u7591\u80fd\u4f7f\u5f97\u6d4b\u8bd5\u7ed3\u679c\u66f4\u597d\u3002<\/p>\n<p>\u6574\u4e2a\u6d4b\u8bd5\u4e2d\uff0c\u90fd\u662f\u7528\u6bcf\u79d2\u949fdelivery\u7684\u6d88\u606f\u7684\u6570\u91cf\u4e58\u4ee5payload\u7684\u957f\u5ea6\u6765\u8ba1\u7b97MB\/second\u7684\uff0c\u6ca1\u6709\u628a\u6d88\u606f\u7684\u5143\u4fe1\u606f\u7b97\u5728\u5185\uff0c\u6240\u4ee5\u5b9e\u9645\u7684\u7f51\u7edc\u4f7f\u7528\u91cf\u4f1a\u6bd4\u8fd9\u4e2a\u5927\u3002\u5bf9\u4e8e\u672c\u6d4b\u8bd5\u6765\u8bf4\uff0c\u6bcf\u6b21\u8fd8\u9700\u4f20\u8f93\u989d\u5916\u768422\u4e2a\u5b57\u8282\uff0c\u5305\u62ec\u4e00\u4e2a\u53ef\u9009\u7684key\uff0c\u6d88\u606f\u957f\u5ea6\u63cf\u8ff0\uff0cCRC\u7b49\u3002\u53e6\u5916\uff0c\u8fd8\u5305\u542b\u4e00\u4e9b\u8bf7\u6c42\u76f8\u5173\u7684overhead\uff0c\u6bd4\u5982topic\uff0cpartition\uff0cacknowledgement\u7b49\u3002\u8fd9\u5c31\u5bfc\u81f4\u6211\u4eec\u6bd4\u8f83\u96be\u5224\u65ad\u662f\u5426\u5df2\u7ecf\u8fbe\u5230\u7f51\u5361\u6781\u9650\uff0c\u4f46\u662f\u628a\u8fd9\u4e9boverhead\u90fd\u7b97\u5728\u541e\u5410\u7387\u91cc\u9762\u5e94\u8be5\u66f4\u5408\u7406\u4e00\u4e9b\u3002\u56e0\u6b64\uff0c\u6211\u4eec\u5df2\u7ecf\u57fa\u672c\u8fbe\u5230\u4e86\u7f51\u5361\u7684\u6781\u9650\u3002<\/p>\n<p>\u521d\u6b65\u89c2\u5bdf\u6b64\u7ed3\u679c\u4f1a\u8ba4\u4e3a\u5b83\u6bd4\u4eba\u4eec\u6240\u9884\u671f\u7684\u8981\u9ad8\u5f88\u591a\uff0c\u5c24\u5176\u5f53\u8003\u8651\u5230Kafka\u8981\u628a\u6570\u636e\u6301\u4e45\u5316\u5230\u78c1\u76d8\u5f53\u4e2d\u3002\u5b9e\u9645\u4e0a\uff0c\u5982\u679c\u4f7f\u7528\u968f\u673a\u8bbf\u95ee\u6570\u636e\u7cfb\u7edf\uff0c\u6bd4\u5982RDBMS\uff0c\u6216\u8005key-velue store\uff0c\u53ef\u9884\u671f\u7684\u6700\u9ad8\u8bbf\u95ee\u9891\u7387\u5927\u6982\u662f5000\u523050000\u4e2a\u8bf7\u6c42\u6bcf\u79d2\uff0c\u8fd9\u548c\u4e00\u4e2a\u597d\u7684RPC\u5c42\u6240\u80fd\u63a5\u53d7\u7684\u8fdc\u7a0b\u8bf7\u6c42\u91cf\u5dee\u4e0d\u591a\u3002\u800c\u8be5\u6d4b\u8bd5\u4e2d\u8fdc\u8d85\u4e8e\u6b64\u7684\u539f\u56e0\u6709\u4e24\u4e2a\u3002<\/p>\n<\/div>\n<ul>\n<li>Kafka\u786e\u4fdd\u5199\u78c1\u76d8\u7684\u8fc7\u7a0b\u662f\u7ebf\u6027\u78c1\u76d8I\/O\uff0c\u6d4b\u8bd5\u4e2d\u4f7f\u7528\u76846\u5757\u5ec9\u4ef7\u78c1\u76d8\u7ebf\u6027I\/O\u7684\u6700\u5927\u541e\u5410\u91cf\u662f822MB\/second\uff0c\u8fd9\u5df2\u7ecf\u8fdc\u5927\u4e8e1Gb\u7f51\u5361\u6240\u80fd\u5e26\u6765\u7684\u541e\u5410\u91cf\u4e86\u3002\u8bb8\u591a\u6d88\u606f\u7cfb\u7edf\u628a\u6570\u636e\u6301\u4e45\u5316\u5230\u78c1\u76d8\u5f53\u6210\u662f\u4e00\u4e2a\u5f00\u9500\u5f88\u5927\u7684\u4e8b\u60c5\uff0c\u8fd9\u662f\u56e0\u4e3a\u4ed6\u4eec\u5bf9\u78c1\u76d8\u7684\u64cd\u4f5c\u90fd\u4e0d\u662f\u7ebf\u6027I\/O\u3002<\/li>\n<li>\u5728\u6bcf\u4e00\u4e2a\u9636\u6bb5\uff0cKafka\u90fd\u5c3d\u91cf\u4f7f\u7528\u6279\u91cf\u5904\u7406\u3002\u5982\u679c\u60f3\u4e86\u89e3\u6279\u5904\u7406\u5728I\/O\u64cd\u4f5c\u4e2d\u7684\u91cd\u8981\u6027\uff0c\u53ef\u4ee5\u53c2\u8003David Patterson\u7684\u201d\u00a0<a href=\"http:\/\/www.ll.mit.edu\/HPEC\/agendas\/proc04\/invited\/patterson_keynote.pdf\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">Latency Lags Bandwidth<\/a>\u00a0\u201c<\/li>\n<\/ul>\n<h3>1\u4e2aproducer\u7ebf\u7a0b\uff0c3\u4e2a\u5f02\u6b65replication<\/h3>\n<div>\n<p>\u8be5\u9879\u6d4b\u8bd5\u4e0e\u4e0a\u4e00\u6d4b\u8bd5\u57fa\u672c\u4e00\u6837\uff0c\u552f\u4e00\u7684\u533a\u522b\u662f\u6bcf\u4e2apartition\u67093\u4e2areplica\uff08\u6240\u4ee5\u7f51\u7edc\u4f20\u8f93\u7684\u548c\u5199\u5165\u78c1\u76d8\u7684\u603b\u7684\u6570\u636e\u91cf\u589e\u52a0\u4e863\u500d\uff09\u3002\u6bcf\u4e00\u4e2abroker\u5373\u8981\u5199\u4f5c\u4e3aleader\u7684partition\uff0c\u4e5f\u8981\u8bfb\uff08\u4eceleader\u8bfb\u6570\u636e\uff09\u5199\uff08\u5c06\u6570\u636e\u5199\u5230\u78c1\u76d8\uff09\u4f5c\u4e3afollower\u7684partition\u3002\u6d4b\u8bd5\u7ed3\u679c\u4e3a\u00a0<strong><em>786,980 records\/second<\/em><\/strong>\uff08\u00a0<strong><em>75.1MB\/second<\/em>\u00a0<\/strong>\uff09\u3002<\/p>\n<p>\u8be5\u9879\u6d4b\u8bd5\u4e2dreplication\u662f\u5f02\u6b65\u7684\uff0c\u4e5f\u5c31\u662f\u8bf4broker\u6536\u5230\u6570\u636e\u5e76\u5199\u5165\u672c\u5730\u78c1\u76d8\u540e\u5c31acknowledge producer\uff0c\u800c\u4e0d\u5fc5\u7b49\u6240\u6709replica\u90fd\u5b8c\u6210replication\u3002\u4e5f\u5c31\u662f\u8bf4\uff0c\u5982\u679cleader crash\u4e86\uff0c\u53ef\u80fd\u4f1a\u4e22\u6389\u4e00\u4e9b\u6700\u65b0\u7684\u8fd8\u672a\u5907\u4efd\u7684\u6570\u636e\u3002\u4f46\u8fd9\u4e5f\u4f1a\u8ba9message acknowledgement\u5ef6\u8fdf\u66f4\u5c11\uff0c\u5b9e\u65f6\u6027\u66f4\u597d\u3002<\/p>\n<p>\u8fd9\u9879\u6d4b\u8bd5\u8bf4\u660e\uff0creplication\u53ef\u4ee5\u5f88\u5feb\u3002\u6574\u4e2a\u96c6\u7fa4\u7684\u5199\u80fd\u529b\u53ef\u80fd\u4f1a\u7531\u4e8e3\u500d\u7684replication\u800c\u53ea\u6709\u539f\u6765\u7684\u4e09\u5206\u4e4b\u4e00\uff0c\u4f46\u662f\u5bf9\u4e8e\u6bcf\u4e00\u4e2aproducer\u6765\u8bf4\u541e\u5410\u7387\u4f9d\u7136\u8db3\u591f\u597d\u3002<\/p>\n<\/div>\n<h3>1\u4e2aproducer\u7ebf\u7a0b\uff0c3\u4e2a\u540c\u6b65replication<\/h3>\n<div>\n<p>\u8be5\u9879\u6d4b\u8bd5\u4e0e\u4e0a\u4e00\u6d4b\u8bd5\u7684\u552f\u4e00\u533a\u522b\u662freplication\u662f\u540c\u6b65\u7684\uff0c\u6bcf\u6761\u6d88\u606f\u53ea\u6709\u5728\u88ab\u00a0<code>in sync<\/code>\u00a0\u96c6\u5408\u91cc\u7684\u6240\u6709replica\u90fd\u590d\u5236\u8fc7\u53bb\u540e\u624d\u4f1a\u88ab\u7f6e\u4e3acommitted\uff08\u6b64\u65f6broker\u4f1a\u5411producer\u53d1\u9001acknowledgement\uff09\u3002\u5728\u8fd9\u79cd\u6a21\u5f0f\u4e0b\uff0cKafka\u53ef\u4ee5\u4fdd\u8bc1\u5373\u4f7fleader crash\u4e86\uff0c\u4e5f\u4e0d\u4f1a\u6709\u6570\u636e\u4e22\u5931\u3002\u6d4b\u8bd5\u7ed3\u679c\u4e3a\u00a0<strong><em>421,823 records\/second<\/em>\u00a0<\/strong>\uff08\u00a0<strong><em>40.2MB\/second<\/em>\u00a0<\/strong>\uff09\u3002<\/p>\n<p>Kafka\u540c\u6b65\u590d\u5236\u4e0e\u5f02\u6b65\u590d\u5236\u5e76\u6ca1\u6709\u672c\u8d28\u7684\u4e0d\u540c\u3002leader\u4f1a\u59cb\u7ec8track follower replica\u4ece\u800c\u76d1\u63a7\u5b83\u4eec\u662f\u5426\u8fd8alive\uff0c\u53ea\u6709\u6240\u6709\u00a0<code>in sync<\/code>\u00a0\u96c6\u5408\u91cc\u7684replica\u90fdacknowledge\u7684\u6d88\u606f\u624d\u53ef\u80fd\u88abconsumer\u6240\u6d88\u8d39\u3002\u800c\u5bf9follower\u7684\u7b49\u5f85\u5f71\u54cd\u4e86\u541e\u5410\u7387\u3002\u53ef\u4ee5\u901a\u8fc7\u589e\u5927batch size\u6765\u6539\u5584\u8fd9\u79cd\u60c5\u51b5\uff0c\u4f46\u4e3a\u4e86\u907f\u514d\u7279\u5b9a\u7684\u4f18\u5316\u800c\u5f71\u54cd\u6d4b\u8bd5\u7ed3\u679c\u7684\u53ef\u6bd4\u6027\uff0c\u672c\u6b21\u6d4b\u8bd5\u5e76\u6ca1\u6709\u505a\u8fd9\u79cd\u8c03\u6574\u3002<\/p>\n<\/div>\n<h3>3\u4e2aproducer,3\u4e2a\u5f02\u6b65replication<\/h3>\n<p>\u8be5\u6d4b\u8bd5\u76f8\u5f53\u4e8e\u628a\u4e0a\u6587\u4e2d\u76841\u4e2aproducer,\u590d\u5236\u5230\u4e863\u53f0\u4e0d\u540c\u7684\u673a\u5668\u4e0a\uff08\u57281\u53f0\u673a\u5668\u4e0a\u8dd1\u591a\u4e2a\u5b9e\u4f8b\u5bf9\u541e\u5410\u7387\u7684\u589e\u52a0\u4e0d\u4f1a\u6709\u592a\u5927\u5e2e\u5fd9\uff0c\u56e0\u4e3a\u7f51\u5361\u5df2\u7ecf\u57fa\u672c\u9971\u548c\u4e86\uff09\uff0c\u8fd93\u4e2aproducer\u540c\u65f6\u53d1\u9001\u6570\u636e\u3002\u6574\u4e2a\u96c6\u7fa4\u7684\u541e\u5410\u7387\u4e3a\u00a0<strong><em>2,024,032 records\/second<\/em>\u00a0<\/strong>\uff08\u00a0<strong><em>193,0MB\/second<\/em>\u00a0<\/strong>\uff09\u3002<\/p>\n<h2>Producer Throughput Vs. Stored Data<\/h2>\n<div>\n<p>\u6d88\u606f\u7cfb\u7edf\u7684\u4e00\u4e2a\u6f5c\u5728\u7684\u5371\u9669\u662f\u5f53\u6570\u636e\u80fd\u90fd\u5b58\u4e8e\u5185\u5b58\u65f6\u6027\u80fd\u5f88\u597d\uff0c\u4f46\u5f53\u6570\u636e\u91cf\u592a\u5927\u65e0\u6cd5\u5b8c\u5168\u5b58\u4e8e\u5185\u5b58\u4e2d\u65f6\uff08\u7136\u540e\u5f88\u591a\u6d88\u606f\u7cfb\u7edf\u90fd\u4f1a\u5220\u9664\u5df2\u7ecf\u88ab\u6d88\u8d39\u7684\u6570\u636e\uff0c\u4f46\u5f53\u6d88\u8d39\u901f\u5ea6\u6bd4\u751f\u4ea7\u901f\u5ea6\u6162\u65f6\uff0c\u4ecd\u4f1a\u9020\u6210\u6570\u636e\u7684\u5806\u79ef\uff09\uff0c\u6570\u636e\u4f1a\u88ab\u8f6c\u79fb\u5230\u78c1\u76d8\uff0c\u4ece\u800c\u4f7f\u5f97\u541e\u5410\u7387\u4e0b\u964d\uff0c\u8fd9\u53c8\u53cd\u8fc7\u6765\u9020\u6210\u7cfb\u7edf\u65e0\u6cd5\u53ca\u65f6\u63a5\u6536\u6570\u636e\u3002\u8fd9\u6837\u5c31\u975e\u5e38\u7cdf\u7cd5\uff0c\u800c\u5b9e\u9645\u4e0a\u5f88\u591a\u60c5\u666f\u4e0b\u4f7f\u7528queue\u7684\u76ee\u7684\u5c31\u662f\u89e3\u51b3\u6570\u636e\u6d88\u8d39\u901f\u5ea6\u548c\u751f\u4ea7\u901f\u5ea6\u4e0d\u4e00\u81f4\u7684\u95ee\u9898\u3002<\/p>\n<p>\u4f46Kafka\u4e0d\u5b58\u5728\u8fd9\u4e00\u95ee\u9898\uff0c\u56e0\u4e3aKafka\u59cb\u7ec8\u4ee5O\uff081\uff09\u7684\u65f6\u95f4\u590d\u6742\u5ea6\u5c06\u6570\u636e\u6301\u4e45\u5316\u5230\u78c1\u76d8\uff0c\u6240\u4ee5\u5176\u541e\u5410\u7387\u4e0d\u53d7\u78c1\u76d8\u4e0a\u6240\u5b58\u50a8\u7684\u6570\u636e\u91cf\u7684\u5f71\u54cd\u3002\u4e3a\u4e86\u9a8c\u8bc1\u8fd9\u4e00\u7279\u6027\uff0c\u505a\u4e86\u4e00\u4e2a\u957f\u65f6\u95f4\u7684\u5927\u6570\u636e\u91cf\u7684\u6d4b\u8bd5\uff0c\u4e0b\u56fe\u662f\u541e\u5410\u7387\u4e0e\u6570\u636e\u91cf\u5927\u5c0f\u7684\u5173\u7cfb\u56fe\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1512\" rel=\"attachment wp-att-1512\"><img loading=\"lazy\" class=\"alignnone size-full wp-image-1512\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/19.png\" alt=\"\" width=\"723\" height=\"390\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/19.png 723w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/19-300x162.png 300w\" sizes=\"(max-width: 723px) 100vw, 723px\" \/><\/a><br \/>\n\u4e0a\u56fe\u4e2d\u6709\u4e00\u4e9bvariance\u7684\u5b58\u5728\uff0c\u5e76\u53ef\u4ee5\u660e\u663e\u770b\u5230\uff0c\u541e\u5410\u7387\u5e76\u4e0d\u53d7\u78c1\u76d8\u4e0a\u6240\u5b58\u6570\u636e\u91cf\u5927\u5c0f\u7684\u5f71\u54cd\u3002\u5b9e\u9645\u4e0a\u4ece\u4e0a\u56fe\u53ef\u4ee5\u770b\u5230\uff0c\u5f53\u78c1\u76d8\u6570\u636e\u91cf\u8fbe\u52301TB\u65f6\uff0c\u541e\u5410\u7387\u548c\u78c1\u76d8\u6570\u636e\u53ea\u6709\u51e0\u767eMB\u65f6\u6ca1\u6709\u660e\u663e\u533a\u522b\u3002<\/p>\n<p>\u8fd9\u4e2avariance\u662f\u7531Linux I\/O\u7ba1\u7406\u9020\u6210\u7684\uff0c\u5b83\u4f1a\u628a\u6570\u636e\u7f13\u5b58\u8d77\u6765\u518d\u6279\u91cfflush\u3002\u4e0a\u56fe\u7684\u6d4b\u8bd5\u7ed3\u679c\u662f\u5728\u751f\u4ea7\u73af\u5883\u4e2d\u5bf9Kafka\u96c6\u7fa4\u505a\u4e86\u4e9btuning\u540e\u5f97\u5230\u7684\uff0c\u8fd9\u4e9btuning\u65b9\u6cd5\u53ef\u53c2\u8003\u00a0<a href=\"http:\/\/kafka.apache.org\/documentation.html#hwandos\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">\u8fd9\u91cc<\/a>\u00a0\u3002<\/p>\n<\/div>\n<h2>consumer\u541e\u5410\u7387<\/h2>\n<p>\u9700\u8981\u6ce8\u610f\u7684\u662f\uff0creplication factor\u5e76\u4e0d\u4f1a\u5f71\u54cdconsumer\u7684\u541e\u5410\u7387\u6d4b\u8bd5\uff0c\u56e0\u4e3aconsumer\u53ea\u4f1a\u4ece\u6bcf\u4e2apartition\u7684leader\u8bfb\u6570\u636e\uff0c\u800c\u4e0ereplicaiton factor\u65e0\u5173\u3002\u540c\u6837\uff0cconsumer\u541e\u5410\u7387\u4e5f\u4e0e\u540c\u6b65\u590d\u5236\u8fd8\u662f\u5f02\u6b65\u590d\u5236\u65e0\u5173\u3002<\/p>\n<h3>1\u4e2aconsumer<\/h3>\n<div>\n<p>\u8be5\u6d4b\u8bd5\u4ece\u67096\u4e2apartition\uff0c3\u4e2areplication\u7684topic\u6d88\u8d3950 million\u7684\u6d88\u606f\u3002\u6d4b\u8bd5\u7ed3\u679c\u4e3a\u00a0<strong><em>940,521 records\/second<\/em>\u00a0<\/strong>\uff08\u00a0<strong><em>89.7MB\/second<\/em>\u00a0<\/strong>\uff09\u3002<\/p>\n<p>\u53ef\u4ee5\u770b\u5230\uff0cKafkar\u7684consumer\u662f\u975e\u5e38\u9ad8\u6548\u7684\u3002\u5b83\u76f4\u63a5\u4ecebroker\u7684\u6587\u4ef6\u7cfb\u7edf\u91cc\u8bfb\u53d6\u6587\u4ef6\u5757\u3002Kafka\u4f7f\u7528\u00a0<a href=\"http:\/\/www.ibm.com\/developerworks\/library\/j-zerocopy\/\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">sendfile API<\/a>\u00a0\u6765\u76f4\u63a5\u901a\u8fc7\u64cd\u4f5c\u7cfb\u7edf\u76f4\u63a5\u4f20\u8f93\uff0c\u800c\u4e0d\u7528\u628a\u6570\u636e\u62f7\u8d1d\u5230\u7528\u6237\u7a7a\u95f4\u3002\u8be5\u9879\u6d4b\u8bd5\u5b9e\u9645\u4e0a\u4ecelog\u7684\u8d77\u59cb\u5904\u5f00\u59cb\u8bfb\u6570\u636e\uff0c\u6240\u4ee5\u5b83\u505a\u4e86\u771f\u5b9e\u7684I\/O\u3002\u5728\u751f\u4ea7\u73af\u5883\u4e0b\uff0cconsumer\u53ef\u4ee5\u76f4\u63a5\u8bfb\u53d6producer\u521a\u521a\u5199\u4e0b\u7684\u6570\u636e\uff08\u5b83\u53ef\u80fd\u8fd8\u5728\u7f13\u5b58\u4e2d\uff09\u3002\u5b9e\u9645\u4e0a\uff0c\u5982\u679c\u5728\u751f\u4ea7\u73af\u5883\u4e0b\u8dd1\u00a0<a href=\"http:\/\/en.wikipedia.org\/wiki\/Iostat\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">I\/O stat<\/a>\u00a0\uff0c\u4f60\u53ef\u4ee5\u770b\u5230\u57fa\u672c\u4e0a\u6ca1\u6709\u7269\u7406\u201c\u8bfb\u201d\u3002\u4e5f\u5c31\u662f\u8bf4\u751f\u4ea7\u73af\u5883\u4e0bconsumer\u7684\u541e\u5410\u7387\u4f1a\u6bd4\u8be5\u9879\u6d4b\u8bd5\u4e2d\u7684\u8981\u9ad8\u3002<\/p>\n<\/div>\n<h3>3\u4e2aconsumer<\/h3>\n<div>\n<p>\u5c06\u4e0a\u9762\u7684consumer\u590d\u5236\u52303\u53f0\u4e0d\u540c\u7684\u673a\u5668\u4e0a\uff0c\u5e76\u4e14\u5e76\u884c\u8fd0\u884c\u5b83\u4eec\uff08\u4ece\u540c\u4e00\u4e2atopic\u4e0a\u6d88\u8d39\u6570\u636e\uff09\u3002\u6d4b\u8bd5\u7ed3\u679c\u4e3a\u00a0<strong><em>2,615,968 records\/second<\/em>\u00a0<\/strong>\uff08\u00a0<strong><em>249.5MB\/second<\/em>\u00a0<\/strong>\uff09\u3002<\/p>\n<p>\u6b63\u5982\u6240\u9884\u671f\u7684\u90a3\u6837\uff0cconsumer\u7684\u541e\u5410\u7387\u51e0\u4e4e\u7ebf\u6027\u589e\u6da8\u3002<\/p>\n<\/div>\n<h2>Producer and Consumer<\/h2>\n<div>\n<p>\u4e0a\u9762\u7684\u6d4b\u8bd5\u53ea\u662f\u628aproducer\u548cconsumer\u5206\u5f00\u6d4b\u8bd5\uff0c\u800c\u8be5\u9879\u6d4b\u8bd5\u540c\u65f6\u8fd0\u884cproducer\u548cconsumer\uff0c\u8fd9\u66f4\u63a5\u8fd1\u4f7f\u7528\u573a\u666f\u3002\u5b9e\u9645\u4e0a\u76ee\u524d\u7684replication\u7cfb\u7edf\u4e2dfollower\u5c31\u76f8\u5f53\u4e8econsumer\u5728\u5de5\u4f5c\u3002<\/p>\n<p>\u8be5\u9879\u6d4b\u8bd5\uff0c\u5728\u5177\u67096\u4e2apartition\u548c3\u4e2areplica\u7684topic\u4e0a\u540c\u65f6\u4f7f\u75281\u4e2aproducer\u548c1\u4e2aconsumer\uff0c\u5e76\u4e14\u4f7f\u7528\u5f02\u6b65\u590d\u5236\u3002\u6d4b\u8bd5\u7ed3\u679c\u4e3a\u00a0<strong><em>795,064 records\/second<\/em>\u00a0<\/strong>\uff08\u00a0<strong><em>75.8MB\/second<\/em>\u00a0<\/strong>\uff09\u3002<\/p>\n<p>\u53ef\u4ee5\u770b\u5230\uff0c\u8be5\u9879\u6d4b\u8bd5\u7ed3\u679c\u4e0e\u5355\u72ec\u6d4b\u8bd51\u4e2aproducer\u65f6\u7684\u7ed3\u679c\u51e0\u4e4e\u4e00\u81f4\u3002\u6240\u4ee5\u8bf4consumer\u975e\u5e38\u8f7b\u91cf\u7ea7\u3002<\/p>\n<\/div>\n<h2>\u6d88\u606f\u957f\u5ea6\u5bf9\u541e\u5410\u7387\u7684\u5f71\u54cd<\/h2>\n<div>\n<p>\u4e0a\u9762\u7684\u6240\u6709\u6d4b\u8bd5\u90fd\u57fa\u4e8e\u77ed\u6d88\u606f\uff08payload 100\u5b57\u8282\uff09\uff0c\u800c\u6b63\u5982\u4e0a\u6587\u6240\u8bf4\uff0c\u77ed\u6d88\u606f\u5bf9Kafka\u6765\u8bf4\u662f\u66f4\u96be\u5904\u7406\u7684\u4f7f\u7528\u65b9\u5f0f\uff0c\u53ef\u4ee5\u9884\u671f\uff0c\u968f\u7740\u6d88\u606f\u957f\u5ea6\u7684\u589e\u5927\uff0crecords\/second\u4f1a\u51cf\u5c0f\uff0c\u4f46MB\/second\u4f1a\u6709\u6240\u63d0\u9ad8\u3002\u4e0b\u56fe\u662frecords\/second\u4e0e\u6d88\u606f\u957f\u5ea6\u7684\u5173\u7cfb\u56fe\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1513\" rel=\"attachment wp-att-1513\"><img loading=\"lazy\" class=\"alignnone size-full wp-image-1513\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/20.png\" alt=\"\" width=\"723\" height=\"393\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/20.png 723w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/20-300x163.png 300w\" sizes=\"(max-width: 723px) 100vw, 723px\" \/><\/a><br \/>\n\u6b63\u5982\u6211\u4eec\u6240\u9884\u671f\u7684\u90a3\u6837\uff0c\u968f\u7740\u6d88\u606f\u957f\u5ea6\u7684\u589e\u52a0\uff0c\u6bcf\u79d2\u949f\u6240\u80fd\u53d1\u9001\u7684\u6d88\u606f\u7684\u6570\u91cf\u9010\u6e10\u51cf\u5c0f\u3002\u4f46\u662f\u5982\u679c\u770b\u6bcf\u79d2\u949f\u53d1\u9001\u7684\u6d88\u606f\u7684\u603b\u5927\u5c0f\uff0c\u5b83\u4f1a\u968f\u7740\u6d88\u606f\u957f\u5ea6\u7684\u589e\u52a0\u800c\u589e\u52a0\uff0c\u5982\u4e0b\u56fe\u6240\u793a\u3002<\/p>\n<p><a href=\"https:\/\/yanjingang.com\/blog\/?attachment_id=1514\" rel=\"attachment wp-att-1514\"><img loading=\"lazy\" class=\"alignnone size-full wp-image-1514\" src=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/21.png\" alt=\"\" width=\"718\" height=\"389\" srcset=\"https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/21.png 718w, https:\/\/yanjingang.com\/blog\/wp-content\/uploads\/2018\/03\/21-300x163.png 300w\" sizes=\"(max-width: 718px) 100vw, 718px\" \/><\/a><br \/>\n\u4ece\u4e0a\u56fe\u53ef\u4ee5\u770b\u51fa\uff0c\u5f53\u6d88\u606f\u957f\u5ea6\u4e3a10\u5b57\u8282\u65f6\uff0c\u56e0\u4e3a\u8981\u9891\u7e41\u5165\u961f\uff0c\u82b1\u4e86\u592a\u591a\u65f6\u95f4\u83b7\u53d6\u9501\uff0cCPU\u6210\u4e86\u74f6\u9888\uff0c\u5e76\u4e0d\u80fd\u5145\u5206\u5229\u7528\u5e26\u5bbd\u3002\u4f46\u4ece100\u5b57\u8282\u5f00\u59cb\uff0c\u6211\u4eec\u53ef\u4ee5\u770b\u5230\u5e26\u5bbd\u7684\u4f7f\u7528\u9010\u6e10\u8d8b\u4e8e\u9971\u548c\uff08\u867d\u7136MB\/second\u8fd8\u662f\u4f1a\u968f\u7740\u6d88\u606f\u957f\u5ea6\u7684\u589e\u52a0\u800c\u589e\u52a0\uff0c\u4f46\u589e\u52a0\u7684\u5e45\u5ea6\u4e5f\u8d8a\u6765\u8d8a\u5c0f\uff09\u3002<\/p>\n<\/div>\n<h2>\u7aef\u5230\u7aef\u7684Latency<\/h2>\n<div>\n<p>\u4e0a\u6587\u4e2d\u8ba8\u8bba\u4e86\u541e\u5410\u7387\uff0c\u90a3\u6d88\u606f\u4f20\u8f93\u7684latency\u5982\u4f55\u5462\uff1f\u4e5f\u5c31\u662f\u8bf4\u6d88\u606f\u4eceproducer\u5230consumer\u9700\u8981\u591a\u5c11\u65f6\u95f4\u5462\uff1f\u8be5\u9879\u6d4b\u8bd5\u521b\u5efa1\u4e2aproducer\u548c1\u4e2aconsumer\u5e76\u53cd\u590d\u8ba1\u65f6\u3002\u7ed3\u679c\u662f\uff0c\u00a0<em>2 ms (median), 3ms (99th percentile, 14ms (99.9th percentile)<\/em>\u00a0\u3002<\/p>\n<p>\uff08\u8fd9\u91cc\u5e76\u6ca1\u6709\u8bf4\u660etopic\u6709\u591a\u5c11\u4e2apartition\uff0c\u4e5f\u6ca1\u6709\u8bf4\u660e\u6709\u591a\u5c11\u4e2areplica\uff0creplication\u662f\u540c\u6b65\u8fd8\u662f\u5f02\u6b65\u3002\u5b9e\u9645\u4e0a\u8fd9\u4f1a\u6781\u5927\u5f71\u54cdproducer\u53d1\u9001\u7684\u6d88\u606f\u88abcommit\u7684latency\uff0c\u800c\u53ea\u6709committed\u7684\u6d88\u606f\u624d\u80fd\u88abconsumer\u6240\u6d88\u8d39\uff0c\u6240\u4ee5\u5b83\u4f1a\u6700\u7ec8\u5f71\u54cd\u7aef\u5230\u7aef\u7684latency\uff09<\/p>\n<\/div>\n<h2>\u91cd\u73b0\u8be5benchmark<\/h2>\n<div>\n<p>\u5982\u679c\u8bfb\u8005\u60f3\u8981\u5728\u81ea\u5df1\u7684\u673a\u5668\u4e0a\u91cd\u73b0\u672c\u6b21benchmark\u6d4b\u8bd5\uff0c\u53ef\u4ee5\u53c2\u8003\u00a0<a href=\"https:\/\/gist.github.com\/jkreps\/c7ddb4041ef62a900e6c\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">\u672c\u6b21\u6d4b\u8bd5\u7684\u914d\u7f6e\u548c\u6240\u4f7f\u7528\u7684\u547d\u4ee4<\/a>\u00a0\u3002<\/p>\n<p>\u5b9e\u9645\u4e0aKafka Distribution\u63d0\u4f9b\u4e86producer\u6027\u80fd\u6d4b\u8bd5\u5de5\u5177\uff0c\u53ef\u901a\u8fc7\u00a0<code>bin\/kafka-producer-perf-test.sh<\/code>\u00a0\u811a\u672c\u6765\u542f\u52a8\u3002\u6240\u4f7f\u7528\u7684\u547d\u4ee4\u5982\u4e0b<\/p>\n<\/div>\n<pre class=\"prettyprint hljs dockerfile\"><code>Producer\r\nSetup\r\nbin\/kafka-topics.sh --zookeeper esv4-hcl197.<span class=\"keyword\">grid<\/span>.linkedin.com:<span class=\"number\"><span class=\"hljs-number\">2181<\/span><\/span> --create --topic test-rep-one --partitions <span class=\"number\"><span class=\"hljs-number\">6<\/span><\/span> --replication-factor <span class=\"number\"><span class=\"hljs-number\">1<\/span><\/span>\r\nbin\/kafka-topics.sh --zookeeper esv4-hcl197.<span class=\"keyword\">grid<\/span>.linkedin.com:<span class=\"number\"><span class=\"hljs-number\">2181<\/span><\/span> --create --topic test --partitions <span class=\"number\"><span class=\"hljs-number\">6<\/span><\/span> --replication-factor <span class=\"number\"><span class=\"hljs-number\">3<\/span><\/span>\r\n\r\nSingle thread, no replication\r\n\r\nbin\/kafka-<span class=\"hljs-keyword\">run<\/span><span class=\"bash\">-class.sh org.apache.kafka.clients.tools.ProducerPerformance <span class=\"hljs-built_in\">test<\/span>7 <\/span><span class=\"number\"><span class=\"bash\">50000000<\/span><\/span> <span class=\"number\"><span class=\"bash\">100<\/span><\/span><span class=\"bash\"> -<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> acks=<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> bootstrap.servers=esv4-hcl198.<\/span><span class=\"keyword\"><span class=\"bash\">grid<\/span><\/span><span class=\"bash\">.linkedin.com:<\/span><span class=\"number\"><span class=\"bash\">9092<\/span><\/span><span class=\"bash\"> buffer.<\/span><span class=\"keyword\"><span class=\"bash\">memory<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">67108864<\/span><\/span><span class=\"bash\"> batch.<\/span><span class=\"keyword\"><span class=\"bash\">size<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">8196<\/span><\/span>\r\n\r\nSingle-thread, async <span class=\"number\"><span class=\"hljs-number\">3<\/span><\/span>x replication\r\n\r\nbin\/kafktopics.sh --zookeeper esv4-hcl197.<span class=\"keyword\">grid<\/span>.linkedin.com:<span class=\"number\"><span class=\"hljs-number\">2181<\/span><\/span> --create --topic test --partitions <span class=\"number\"><span class=\"hljs-number\">6<\/span><\/span> --replication-factor <span class=\"number\"><span class=\"hljs-number\">3<\/span><\/span>\r\nbin\/kafka-<span class=\"hljs-keyword\">run<\/span><span class=\"bash\">-class.sh org.apache.kafka.clients.tools.ProducerPerformance <span class=\"hljs-built_in\">test<\/span>6 <\/span><span class=\"number\"><span class=\"bash\">50000000<\/span><\/span> <span class=\"number\"><span class=\"bash\">100<\/span><\/span><span class=\"bash\"> -<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> acks=<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> bootstrap.servers=esv4-hcl198.<\/span><span class=\"keyword\"><span class=\"bash\">grid<\/span><\/span><span class=\"bash\">.linkedin.com:<\/span><span class=\"number\"><span class=\"bash\">9092<\/span><\/span><span class=\"bash\"> buffer.<\/span><span class=\"keyword\"><span class=\"bash\">memory<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">67108864<\/span><\/span><span class=\"bash\"> batch.<\/span><span class=\"keyword\"><span class=\"bash\">size<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">8196<\/span><\/span>\r\n\r\nSingle-thread, sync <span class=\"number\"><span class=\"hljs-number\">3<\/span><\/span>x replication\r\n\r\nbin\/kafka-<span class=\"hljs-keyword\">run<\/span><span class=\"bash\">-class.sh org.apache.kafka.clients.tools.ProducerPerformance <span class=\"hljs-built_in\">test<\/span> <\/span><span class=\"number\"><span class=\"bash\">50000000<\/span><\/span> <span class=\"number\"><span class=\"bash\">100<\/span><\/span><span class=\"bash\"> -<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> acks=-<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> bootstrap.servers=esv4-hcl198.<\/span><span class=\"keyword\"><span class=\"bash\">grid<\/span><\/span><span class=\"bash\">.linkedin.com:<\/span><span class=\"number\"><span class=\"bash\">9092<\/span><\/span><span class=\"bash\"> buffer.<\/span><span class=\"keyword\"><span class=\"bash\">memory<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">67108864<\/span><\/span><span class=\"bash\"> batch.<\/span><span class=\"keyword\"><span class=\"bash\">size<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">64000<\/span><\/span>\r\n\r\nThree Producers, <span class=\"number\"><span class=\"hljs-number\">3<\/span><\/span>x async replication\r\nbin\/kafka-<span class=\"hljs-keyword\">run<\/span><span class=\"bash\">-class.sh org.apache.kafka.clients.tools.ProducerPerformance <span class=\"hljs-built_in\">test<\/span> <\/span><span class=\"number\"><span class=\"bash\">50000000<\/span><\/span> <span class=\"number\"><span class=\"bash\">100<\/span><\/span><span class=\"bash\"> -<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> acks=<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> bootstrap.servers=esv4-hcl198.<\/span><span class=\"keyword\"><span class=\"bash\">grid<\/span><\/span><span class=\"bash\">.linkedin.com:<\/span><span class=\"number\"><span class=\"bash\">9092<\/span><\/span><span class=\"bash\"> buffer.<\/span><span class=\"keyword\"><span class=\"bash\">memory<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">67108864<\/span><\/span><span class=\"bash\"> batch.<\/span><span class=\"keyword\"><span class=\"bash\">size<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">8196<\/span><\/span>\r\n\r\nThroughput Versus Stored Data\r\n\r\nbin\/kafka-<span class=\"hljs-keyword\">run<\/span><span class=\"bash\">-class.sh org.apache.kafka.clients.tools.ProducerPerformance <span class=\"hljs-built_in\">test<\/span> <\/span><span class=\"number\"><span class=\"bash\">50000000000<\/span><\/span> <span class=\"number\"><span class=\"bash\">100<\/span><\/span><span class=\"bash\"> -<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> acks=<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> bootstrap.servers=esv4-hcl198.<\/span><span class=\"keyword\"><span class=\"bash\">grid<\/span><\/span><span class=\"bash\">.linkedin.com:<\/span><span class=\"number\"><span class=\"bash\">9092<\/span><\/span><span class=\"bash\"> buffer.<\/span><span class=\"keyword\"><span class=\"bash\">memory<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">67108864<\/span><\/span><span class=\"bash\"> batch.<\/span><span class=\"keyword\"><span class=\"bash\">size<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">8196<\/span><\/span>\r\n\r\nEffect of message <span class=\"keyword\">size<\/span>\r\n\r\n<span class=\"keyword\">for<\/span> i <span class=\"keyword\">in<\/span> <span class=\"number\"><span class=\"hljs-number\">10<\/span><\/span> <span class=\"number\"><span class=\"hljs-number\">100<\/span><\/span> <span class=\"number\"><span class=\"hljs-number\">1000<\/span><\/span> <span class=\"number\"><span class=\"hljs-number\">10000<\/span><\/span> <span class=\"number\"><span class=\"hljs-number\">100000<\/span><\/span>;\r\n<span class=\"keyword\">do<\/span>\r\necho <span class=\"string\"><span class=\"hljs-string\">\"\"<\/span><\/span>\r\necho <span class=\"variable\">$i<\/span>\r\nbin\/kafka-<span class=\"hljs-keyword\">run<\/span><span class=\"bash\">-class.sh org.apache.kafka.clients.tools.ProducerPerformance <span class=\"hljs-built_in\">test<\/span> <\/span><span class=\"variable\"><span class=\"bash\">$(<\/span><\/span><span class=\"bash\">(<\/span><span class=\"number\"><span class=\"bash\">1000<\/span><\/span><span class=\"variable\"><span class=\"bash\">*1024<\/span><\/span><span class=\"variable\"><span class=\"bash\">*1024<\/span><\/span><span class=\"bash\">\/<\/span><span class=\"variable\"><span class=\"bash\"><span class=\"hljs-variable\">$i<\/span><\/span><\/span><span class=\"bash\">)) <\/span><span class=\"variable\"><span class=\"bash\"><span class=\"hljs-variable\">$i<\/span><\/span><\/span><span class=\"bash\"> -<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> acks=<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> bootstrap.servers=esv4-hcl198.<\/span><span class=\"keyword\"><span class=\"bash\">grid<\/span><\/span><span class=\"bash\">.linkedin.com:<\/span><span class=\"number\"><span class=\"bash\">9092<\/span><\/span><span class=\"bash\"> buffer.<\/span><span class=\"keyword\"><span class=\"bash\">memory<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">67108864<\/span><\/span><span class=\"bash\"> batch.<\/span><span class=\"keyword\"><span class=\"bash\">size<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">128000<\/span><\/span>\r\ndone;\r\n\r\nConsumer\r\nConsumer throughput\r\n\r\nbin\/kafka-consumer-perf-test.sh --zookeeper esv4-hcl197.<span class=\"keyword\">grid<\/span>.linkedin.com:<span class=\"number\"><span class=\"hljs-number\">2181<\/span><\/span> --messages <span class=\"number\"><span class=\"hljs-number\">50000000<\/span><\/span> --topic test --threads <span class=\"number\"><span class=\"hljs-number\">1<\/span><\/span>\r\n\r\n<span class=\"number\"><span class=\"hljs-number\">3<\/span><\/span> Consumers\r\n\r\nOn three servers, <span class=\"hljs-keyword\">run<\/span><span class=\"bash\">:\r\n<\/span>bin\/kafka-consumer-perf-test.sh --zookeeper esv4-hcl197.<span class=\"keyword\">grid<\/span>.linkedin.com:<span class=\"number\"><span class=\"hljs-number\">2181<\/span><\/span> --messages <span class=\"number\"><span class=\"hljs-number\">50000000<\/span><\/span> --topic test --threads <span class=\"number\"><span class=\"hljs-number\">1<\/span><\/span>\r\n\r\nEnd-to-end Latency\r\n\r\nbin\/kafka-<span class=\"hljs-keyword\">run<\/span><span class=\"bash\">-class.sh kafka.tools.TestEndToEndLatency esv4-hcl198.<\/span><span class=\"keyword\"><span class=\"bash\">grid<\/span><\/span><span class=\"bash\">.linkedin.com:<\/span><span class=\"number\"><span class=\"bash\">9092<\/span><\/span><span class=\"bash\"> esv4-hcl197.<\/span><span class=\"keyword\"><span class=\"bash\">grid<\/span><\/span><span class=\"bash\">.linkedin.com:<\/span><span class=\"number\"><span class=\"bash\">2181<\/span><\/span><span class=\"bash\"> <span class=\"hljs-built_in\">test<\/span> <\/span><span class=\"number\"><span class=\"bash\">5000<\/span><\/span>\r\n\r\nProducer and consumer\r\n\r\nbin\/kafka-<span class=\"hljs-keyword\">run<\/span><span class=\"bash\">-class.sh org.apache.kafka.clients.tools.ProducerPerformance <span class=\"hljs-built_in\">test<\/span> <\/span><span class=\"number\"><span class=\"bash\">50000000<\/span><\/span> <span class=\"number\"><span class=\"bash\">100<\/span><\/span><span class=\"bash\"> -<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> acks=<\/span><span class=\"number\"><span class=\"bash\">1<\/span><\/span><span class=\"bash\"> bootstrap.servers=esv4-hcl198.<\/span><span class=\"keyword\"><span class=\"bash\">grid<\/span><\/span><span class=\"bash\">.linkedin.com:<\/span><span class=\"number\"><span class=\"bash\">9092<\/span><\/span><span class=\"bash\"> buffer.<\/span><span class=\"keyword\"><span class=\"bash\">memory<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">67108864<\/span><\/span><span class=\"bash\"> batch.<\/span><span class=\"keyword\"><span class=\"bash\">size<\/span><\/span><span class=\"bash\">=<\/span><span class=\"number\"><span class=\"bash\">8196<\/span><\/span>\r\n\r\nbin\/kafka-consumer-perf-test.sh --zookeeper esv4-hcl197.<span class=\"keyword\">grid<\/span>.linkedin.com:<span class=\"number\"><span class=\"hljs-number\">2181<\/span><\/span> --messages <span class=\"number\"><span class=\"hljs-number\">50000000<\/span><\/span> --topic test --threads <span class=\"number\"><span class=\"hljs-number\">1<\/span><\/span>\r\n<\/code><\/pre>\n<p>broker\u914d\u7f6e\u5982\u4e0b<\/p>\n<pre class=\"prettyprint hljs coffeescript\"><code><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##### Server Basics ###<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The id of the broker. This must be set to a unique integer for each broker.<\/span><\/span>\r\nbroker.id=<span class=\"number\"><span class=\"hljs-number\">0<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##### Socket Server Settings ###<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The port the socket server listens on<\/span><\/span>\r\nport=<span class=\"number\"><span class=\"hljs-number\">9092<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># Hostname the broker will bind to and advertise to producers and consumers.<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># If not set, the server will bind to all interfaces and advertise the value returned from<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># from java.net.InetAddress.getCanonicalHostName().<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\">#host.name=localhost<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The number of threads handling network requests<\/span><\/span>\r\nnum.network.threads=<span class=\"number\"><span class=\"hljs-number\">4<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The number of threads doing disk I\/O<\/span><\/span>\r\nnum.io.threads=<span class=\"number\"><span class=\"hljs-number\">8<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The send buffer (SO_SNDBUF) used by the socket server<\/span><\/span>\r\nsocket.send.buffer.bytes=<span class=\"number\"><span class=\"hljs-number\">1048576<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The receive buffer (SO_RCVBUF) used by the socket server<\/span><\/span>\r\nsocket.receive.buffer.bytes=<span class=\"number\"><span class=\"hljs-number\">1048576<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The maximum size of a request that the socket server will accept (protection against OOM)<\/span><\/span>\r\nsocket.request.max.bytes=<span class=\"number\"><span class=\"hljs-number\">104857600<\/span><\/span>\r\n\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##### Log Basics ###<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The directory under which to store log files<\/span><\/span>\r\nlog.dirs=\/grid\/a\/dfs-data\/kafka-logs,\/grid\/b\/dfs-data\/kafka-logs,\/grid\/c\/dfs-data\/kafka-logs,\/grid\/d\/dfs-data\/kafka-logs,\/grid\/e\/dfs-data\/kafka-logs,\/grid\/f\/dfs-data\/kafka-logs\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The number of logical partitions per topic per server. More partitions allow greater parallelism<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># for consumption, but also mean more files.<\/span><\/span>\r\nnum.partitions=<span class=\"number\"><span class=\"hljs-number\">8<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##### Log Flush Policy ###<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The following configurations control the flush of data to disk. This is the most<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># important performance knob in kafka.<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># There are a few important trade-offs here:<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\">#    1. Durability: Unflushed data is at greater risk of loss in the event of a crash.<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\">#    2. Latency: Data is not made available to consumers until it is flushed (which adds latency).<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\">#    3. Throughput: The flush is generally the most expensive operation. <\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The settings below allow one to configure the flush policy to flush data after a period of time or<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># every N messages (or both). This can be done globally and overridden on a per-topic basis.<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># Per-topic overrides for log.flush.interval.ms<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\">#log.flush.intervals.ms.per.topic=topic1:1000, topic2:3000<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##### Log Retention Policy ###<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The following configurations control the disposal of log segments. The policy can<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># be set to delete segments after a period of time, or after a given size has accumulated.<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># A segment will be deleted whenever *either* of these criteria are met. Deletion always happens<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># from the end of the log.<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The minimum age of a log file to be eligible for deletion<\/span><\/span>\r\nlog.retention.hours=<span class=\"number\"><span class=\"hljs-number\">168<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># A size-based retention policy for logs. Segments are pruned from the log as long as the remaining<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># segments don't drop below log.retention.bytes.<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\">#log.retention.bytes=1073741824<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The maximum size of a log segment file. When this size is reached a new log segment will be created.<\/span><\/span>\r\nlog.segment.bytes=<span class=\"number\"><span class=\"hljs-number\">536870912<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># The interval at which log segments are checked to see if they can be deleted according <\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># to the retention policies<\/span><\/span>\r\nlog.cleanup.interval.mins=<span class=\"number\"><span class=\"hljs-number\">1<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##### Zookeeper ###<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">######<\/span><\/span><span class=\"comment\"><span class=\"hljs-comment\">##<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># Zookeeper connection string (see zookeeper docs for details).<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># This is a comma separated host:port pairs, each corresponding to a zk<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># server. e.g. \"127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002\".<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># You can also append an optional chroot string to the urls to specify the<\/span><\/span>\r\n<span class=\"comment\"><span class=\"hljs-comment\"># root directory for all kafka znodes.<\/span><\/span>\r\nzookeeper.connect=esv4-hcl197.grid.linkedin.<span class=\"attribute\">com<\/span>:<span class=\"number\"><span class=\"hljs-number\">2181<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># Timeout in ms for connecting to zookeeper<\/span><\/span>\r\nzookeeper.connection.timeout.ms=<span class=\"number\"><span class=\"hljs-number\">1000000<\/span><\/span>\r\n\r\n<span class=\"comment\"><span class=\"hljs-comment\"># metrics reporter properties<\/span><\/span>\r\nkafka.metrics.polling.interval.secs=<span class=\"number\"><span class=\"hljs-number\">5<\/span><\/span>\r\nkafka.metrics.reporters=kafka.metrics.KafkaCSVMetricsReporter\r\nkafka.csv.metrics.dir=\/tmp\/kafka_metrics\r\n<span class=\"comment\"><span class=\"hljs-comment\"># Disable csv reporting by default.<\/span><\/span>\r\nkafka.csv.metrics.reporter.enabled=<span class=\"literal\"><span class=\"hljs-literal\">false<\/span><\/span>\r\n\r\nreplica.lag.max.messages=<span class=\"number\"><span class=\"hljs-number\">10000000<\/span><\/span>\r\n<\/code><\/pre>\n<p>\u8bfb\u8005\u4e5f\u53ef\u53c2\u8003\u53e6\u5916\u4e00\u4efd\u00a0<a href=\"http:\/\/liveramp.com\/blog\/kafka-0-8-producer-performance-2\/\" target=\"_blank\" rel=\"nofollow,noindex noopener noreferrer\">Kafka\u6027\u80fd\u6d4b\u8bd5\u62a5\u544a<\/a><\/p>\n<p>\u6458\u81ea\uff1ahttp:\/\/www.jasongj.com\/2015\/01\/02\/Kafka%E6%B7%B1%E5%BA%A6%E8%A7%A3%E6%9E%90\/<\/p>\n","protected":false},"excerpt":{"rendered":"<p>Kafka\u662f\u4e00\u79cd\u5206\u5e03\u5f0f\u7684\uff0c\u57fa\u4e8e\u53d1\u5e03\/\u8ba2\u9605\u7684\u6d88\u606f\u7cfb\u7edf\u3002\u4e3b\u8981\u8bbe\u8ba1\u76ee\u6807\u5982\u4e0b\uff1a<\/p>\n<p>\u4ee5\u65f6\u95f4\u590d\u6742\u5ea6\u4e3aO(1)\u7684\u65b9\u5f0f\u63d0\u4f9b\u6d88\u606f\u6301\u4e45\u5316\u80fd\u529b\uff0c\u5e76\u4fdd\u8bc1\u5373\u4f7f\u5bf9TB\u7ea7\u4ee5\u4e0a\u6570\u636e\u4e5f\u80fd\u4fdd\u8bc1\u5e38\u6570\u65f6\u95f4\u7684\u8bbf\u95ee\u6027\u80fd<br \/>\n\u9ad8\u541e\u5410\u7387\u3002\u5373\u4f7f\u5728\u975e\u5e38\u5ec9\u4ef7\u7684\u5546\u7528\u673a\u5668\u4e0a\u4e5f\u80fd\u505a\u5230\u5355\u673a\u652f\u6301\u6bcf\u79d2100K\u6761\u6d88\u606f\u7684\u4f20\u8f93<br \/>\n\u652f\u6301Kafka Server\u95f4\u7684\u6d88\u606f\u5206\u533a\uff0c\u53ca\u5206\u5e03\u5f0f\u6d88\u606f\u6d88\u8d39\uff0c\u540c\u65f6\u4fdd\u8bc1\u6bcf\u4e2apartition\u5185\u7684\u6d88\u606f\u987a\u5e8f\u4f20\u8f93<br \/>\n\u540c\u65f6\u652f\u6301\u79bb\u7ebf\u6570\u636e\u5904\u7406\u548c\u5b9e\u65f6\u6570\u636e\u5904\u7406<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":[],"categories":[509],"tags":[510],"_links":{"self":[{"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/1493"}],"collection":[{"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1493"}],"version-history":[{"count":0,"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/1493\/revisions"}],"wp:attachment":[{"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1493"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1493"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/yanjingang.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1493"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}