{"id":12046,"date":"2016-11-19T06:00:07","date_gmt":"2016-11-18T21:00:07","guid":{"rendered":"https:\/\/jirak.net\/wp\/amazon-emr-%ed%81%b4%eb%9f%ac%ec%8a%a4%ed%84%b0-%ec%9e%90%eb%8f%99-%ed%99%95%ec%9e%a5-%ea%b8%b0%eb%8a%a5-%ec%b6%94%ea%b0%80\/"},"modified":"2016-11-21T02:34:23","modified_gmt":"2016-11-20T17:34:23","slug":"amazon-emr-%ed%81%b4%eb%9f%ac%ec%8a%a4%ed%84%b0-%ec%9e%90%eb%8f%99-%ed%99%95%ec%9e%a5-%ea%b8%b0%eb%8a%a5-%ec%b6%94%ea%b0%80","status":"publish","type":"post","link":"https:\/\/jirak.net\/wp\/amazon-emr-%ed%81%b4%eb%9f%ac%ec%8a%a4%ed%84%b0-%ec%9e%90%eb%8f%99-%ed%99%95%ec%9e%a5-%ea%b8%b0%eb%8a%a5-%ec%b6%94%ea%b0%80\/","title":{"rendered":"Amazon EMR \ud074\ub7ec\uc2a4\ud130 \uc790\ub3d9 \ud655\uc7a5 \uae30\ub2a5 \ucd94\uac00"},"content":{"rendered":"<p>Amazon EMR \ud074\ub7ec\uc2a4\ud130 \uc790\ub3d9 \ud655\uc7a5 \uae30\ub2a5 \ucd94\uac00<br \/>\n<img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/jirak.net\/wp\/wp-content\/uploads\/2016\/11\/emr_hw_config_cluster_1.png\" width=\"860\" height=\"366\"><\/p>\n<p><a title=\"\" href=\"https:\/\/aws.amazon.com\/elasticmapreduce\/\">Amazon EMR<\/a> \ud300\uc740 \ucd5c\uadfc \uc2e0\uaddc \ubc84\uc804\uc744 \uacc4\uc18d \ucd9c\uc2dc\ud558\uba74\uc11c, \uc774\ubc88 \ubd84\uae30\uc5d0\ub9cc \ub2e4\uc591\ud55c \uae30\ub2a5\uc744 \ucd94\uac00\ud558\uc600\uc2b5\ub2c8\ub2e4.<\/p>\n<ul>\n<li><strong>9\uc6d4<\/strong> \u2013 <a href=\"https:\/\/aws.amazon.com\/about-aws\/whats-new\/2016\/09\/amazon-emr-now-supports-data-encryption-for-apache-spark-tez-and-hadoop-mapreduce\/\">Apache Spark, Tez, Hadoop MapReduce \ub370\uc774\ud130 \uc554\ud638\ud654<\/a>.<\/li>\n<li><strong>9\uc6d4<\/strong> \u2013 <a href=\"https:\/\/aws.amazon.com\/about-aws\/whats-new\/2016\/09\/the-amazon-emr-dynamodb-connector-for-apache-hive-and-apache-spark-is-now-open-source\/\">Apache Hive\ub97c \uc704\ud55c \uc624\ud508 \uc18c\uc2a4 EMR-DynamoDB Connector \uc9c0\uc6d0<\/a>.<\/li>\n<li><strong>11\uc6d4<\/strong> \u2013 <a href=\"https:\/\/aws.amazon.com\/about-aws\/whats-new\/2016\/11\/now-run-real-time-stream-processing-at-scale-with-apache-flink-on-amazon-emr\/\">Apache Flink\uc5d0\uc11c \uc2e4\uc2dc\uac04 \uc2a4\ud2b8\ub9ac\ubc0d \ucc98\ub9ac<\/a>.<\/li>\n<li><strong>11\uc6d4<\/strong> \u2013 <a href=\"https:\/\/aws.amazon.com\/about-aws\/whats-new\/2016\/11\/amazon-emr-now-supports-fine-grained-access-control-using-cluster-tags\/\">\ud074\ub7ec\uc2a4\ud130 \ud0dc\uadf8\ub97c \ud65c\uc6a9\ud55c \uc138\ubd80 \uad00\ub9ac \uae30\ub2a5<\/a>.<\/li>\n<\/ul>\n<p>\uc624\ub298\ubd80\ud130 Amazon EMR \ud074\ub7ec\uc2a4\ud130\uc5d0 \ub300\ud55c \uc790\ub3d9 \ud655\uc7a5 \uae30\ub2a5\uc744 \uc81c\uacf5\ud569\ub2c8\ub2e4. \uc774\uc81c \uc218\ud3c9 \ud655\uc7a5 \ubc0f \uac10\uc18c \uae30\ub2a5\uc744 \uc0ac\uc6a9\ud558\uc5ec \uc791\uc5c5\ub7c9 \ubcc0\uacbd\uc5d0 \ub300\ud55c \ud074\ub7ec\uc2a4\ud130\uc758 \ucf54\uc5b4 \ubc0f \uc791\uc5c5 \ub178\ub4dc \uc218\ub97c \uc870\uc815\ud558\uace0 \uc790\uc6d0 \uc0ac\uc6a9\uc744 \ucd5c\uc801\ud654 \ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n<p><strong>\uc218\ud3c9 \ud655\uc7a5(Scale-out) \uc815\ucc45<\/strong>: \ucef4\ud4e8\ud305 \uc6a9\ub7c9\uc744 \ucd94\uac00\ud558\uc5ec \ub354 \ud070 \ubb38\uc81c\ub97c \ud574\uacb0\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. Apache Spark \ubc0f Apache Hive\uc640 \uac19\uc740 \uc751\uc6a9 \ud504\ub85c\uadf8\ub7a8\uc744 \uc790\ub3d9\uc73c\ub85c \uc99d\uac00\ud558\ub294 \ucc98\ub9ac \ub2a5\ub825\uc744 \ud65c\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n<p><strong>\uc218\ud3c9 \uac10\uc18c(Scale-in) \uc815\ucc45<\/strong>: \uc778\uc2a4\ud134\uc2a4 \uc0ac\uc6a9 \uc2dc\uac04\uc774 \ub05d\ub098\uac70\ub098 \uc804\uccb4 \uc791\uc5c5\uc758 \ub85c\ub4dc\uac00 \uc904\uc5b4\ub4e4\ub54c \uc6a9\ub7c9\uc744 \uc81c\uac70\ud569\ub2c8\ub2e4. YARN \uad00\ub9ac \ucf58\ud14c\uc774\ub108\uc5d0\uc11c \ub178\ub4dc\uac00 \ud558\ub098 \uc904\uc5b4 \ub4e4\uba74, YARN\uc740 \ub2e4\ub978 \ub178\ub4dc\uc5d0\uc11c \uc7ac\uc2e4\ud589\ub429\ub2c8\ub2e4. (\uc790\uc138\ud55c \uc0ac\ud56d\uc740 <a href=\"http:\/\/docs.aws.amazon.com\/ElasticMapReduce\/latest\/ManagementGuide\/emr-scaledown-behavior.html\">Configure Cluster Scale-Down Behavior<\/a> \ubb38\uc11c\ub97c \ucc38\uace0\ud558\uc138\uc694.)<\/p>\n<p><strong>\uc790\ub3d9 \uc2a4\ucf00\uc77c\ub9c1 \uc0ac\uc6a9\ud558\uae30<\/strong><br \/> \uc790\ub3d9 \uc2a4\ucf00\uc77c\ub9c1\uc744 \uc0ac\uc6a9\ud558\ub824\uba74, \uba3c\uc800 EC2 \uc778\uc2a4\ud134\uc2a4\ub97c \uc2dc\uc791\ud558\uace0 \uc885\ub8cc \ud560 \uc218 \uc788\ub294 \uad8c\ud55c\uc744 \ubd80\uc5ec\ud558\ub294 IAM \uc5ed\ud560\uc774 \ud074\ub7ec\uc2a4\ud130\uc640 \uc5f0\uacb0\ub418\uc5b4 \uc788\uc5b4\uc57c\ud569\ub2c8\ub2e4. \uc5ec\ub7ec\ubd84\uc774 EMR \ucf58\uc194\uc5d0\uc11c \ud074\ub7ec\uc2a4\ud130\ub97c \uc791\uc131\ud558\ub294 \uacbd\uc6b0, <code>EMR_AutoScaling_DefaultRole<\/code>\ub97c \ub9cc\ub4ed\ub2c8\ub2e4. \uae30\ubcf8 \uac12 \uc788\ub294 \uadf8\ub300\ub85c \uc0ac\uc6a9\ud558\uac70\ub098 \ud544\uc694\uc5d0 \ub530\ub77c \uc0ac\uc6a9\uc790 \uc815\uc758\ub85c \uc9c0\uc815\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \ud504\ub85c\uadf8\ub798\ubc0d \ubc29\uc2dd \ub610\ub294 \uba85\ub839\uc904\uc744 \ud1b5\ud574 \ud074\ub7ec\uc2a4\ud130\ub97c \ub9cc\ub4dc\ub294 \uacbd\uc6b0, \uc544\ub798\uc640 \uac19\uc774 \uc9c1\uc811 \ub9cc\ub4e4 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \ub354 \uc790\uc138\ud558 \uac83\uc740 <a href=\"http:\/\/docs.aws.amazon.com\/cli\/latest\/reference\/emr\/create-default-roles.html\">create the default roles from the command line<\/a>\uc744 \ucc38\uace0\ud558\uc138\uc694.<\/p>\n<div class=\"prism-show-language\">\n<div class=\"prism-show-language-label\">\n  Bash\n <\/div>\n<\/div>\n<pre class=\" language-bash\"><code class=\" language-bash\">$ aws emr create<span class=\"token operator\">-<\/span>default<span class=\"token operator\">-<\/span>roles<\/code><\/pre>\n<p>\ucf58\uc194\uc5d0\uc11c \ud074\ub7ec\uc2a4\ud130\ub97c \ub9cc\ub4e4 \ub54c \uace0\uae09 \uc635\uc158\uc744 \ud074\ub9ad\ud558\uc5ec \uc790\ub3d9 \ud655\uc7a5 \uc815\ucc45\uc744 \ud3b8\uc9d1 \ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n<p><img loading=\"lazy\" decoding=\"async\" class=\"aligncenter\" src=\"https:\/\/media.amazonwebservices.com\/blog\/2016\/emr_hw_config_cluster_1.png\" alt=\"\" width=\"860\" height=\"366\" \/><\/p>\n<p>\uc5f0\ud544 \uc544\uc774\ucf58\uc744 \ud074\ub9ad\ud558\uba74 \uc815\ucc45 \uc815\ubcf4 \uc218\uc815\uc744 \ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc800\uc758 \uc608\uc81c\ub294 \uc544\ub798\uc640 \uac19\uc2b5\ub2c8\ub2e4.<\/p>\n<p><img loading=\"lazy\" decoding=\"async\" class=\"aligncenter\" src=\"https:\/\/media.amazonwebservices.com\/blog\/2016\/emr_scale_out_policy_2.png\" alt=\"\" width=\"599\" height=\"464\" \/><\/p>\n<p>\ubcf8 \uc815\ucc45\uc774 <code>YARNMemoryAvailablePercentage<\/code>\uc5d0 \uc758\ud574 \uc2e4\ud589\ud558\uae30 \ub54c\ubb38\uc5d0 Spark, TEZ, Hadoop MapReduce \ub4f1\uc758 \ud504\ub808\uc784\uc6cc\ud06c\ub97c \uc2e4\ud589\ud560 \ub54c, \uba54\ubaa8\ub9ac\uac00 \ub0ae\uc740 \uc0c1\ud0dc\uc5d0\uc11c \ud65c\uc131\ud654\ub420 \uac83\uc785\ub2c8\ub2e4. \uc544\ub798 \uc635\uc158\ucc98\ub7fc \ub2e4\uc591\ud55c \ud1b5\uacc4\uce58\ub97c \uc120\ud0dd\ud560 \uc218\ub3c4 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n<p><img loading=\"lazy\" decoding=\"async\" class=\"aligncenter\" src=\"https:\/\/media.amazonwebservices.com\/blog\/2016\/emr_policy_options_1.png\" alt=\"\" width=\"359\" height=\"417\" \/><\/p>\n<p>\uc81c\uac00 \ub9cc\ub4e0 \uc790\ub3d9 \uc2a4\ucf00\uc77c\ub9c1 \uc815\ucc45\uc740 \uc544\ub798\uc640 \uac19\uc2b5\ub2c8\ub2e4.<\/p>\n<p><img loading=\"lazy\" decoding=\"async\" class=\"aligncenter\" src=\"https:\/\/media.amazonwebservices.com\/blog\/2016\/emr_scale_in_policy_1.png\" alt=\"\" width=\"700\" height=\"304\" \/><\/p>\n<p>\ub3d9\uc77c\ud55c \ud1b5\uacc4 \uc14b\uc744 \uc120\ud0dd\ud560 \uc218\ub3c4 \uc788\uace0, \uac01 \uc815\ucc45\uc758 <code>Cooldown period<\/code>(\uc7ac\uc0ac\uc6a9 \ub300\uae30 \uc2dc\uac04)\uc744 \uc124\uc815\ud560 \uc218\ub3c4 \uc788\uc2b5\ub2c8\ub2e4. \uc774 \uac12\uc740 \ud655\uc7a5 \uae30\ub2a5\uc774 \uc6b4\uc601\ub418\ub294 \uc0ac\uc774\uc758 \ucd5c\uc18c \uc2dc\uac04\uc744 \uc124\uc815\ud558\uace0 \ubcc0\uacbd \uc0ac\ud56d\uc774 \uc801\uc6a9\ub420 \ub54c \ud1b5\uacc4\uce58\ub97c \uc548\uc815\uc801\uc73c\ub85c \uc720\uc9c0\ud569\ub2c8\ub2e4.<\/p>\n<p>\uae30\ubcf8 \uc815\ucc45(<code>YARNMemoryAvailablePercentage<\/code> \ubc0f <code>ContainerPendingRatio<\/code>\uc5d0 \uc758\ud574 \uc2e4\ud589)\uc740 \ucf58\uc194\uc5d0\uc11c\ub3c4 \uc0ac\uc6a9 \uac00\ub2a5\ud569\ub2c8\ub2e4.<\/p>\n<p><strong>\uc815\uc2dd \ucd9c\uc2dc<\/strong><br \/> Amazon EMR\uc758 \uc790\ub3d9 \uc2a4\ucf00\uc77c\ub9c1\uc5d0 \ub300\ud55c \uc790\uc138\ud55c \uc815\ubcf4\ub294 <a href=\"http:\/\/docs.aws.amazon.com\/ElasticMapReduce\/latest\/ManagementGuide\/emr-automatic-scaling.html\">Scaling Cluster Resources<\/a>\uc744 \ucc38\uace0\ud558\uc2dc\uace0, \ubcf8 \uae30\ub2a5\uc740 \uc624\ub298 \ubd80\ud130 <strong>emr-5.1.0<\/strong> \ubc84\uc804\uc5d0\uc11c \uc0ac\uc6a9 \uac00\ub2a5\ud569\ub2c8\ub2e4.<\/p>\n<p>\u2014 <a href=\"https:\/\/twitter.com\/jeffbarr\">Jeff<\/a>;<\/p>\n<p><em>\uc774 \uae00\uc740 <a href=\"https:\/\/aws.amazon.com\/blogs\/aws\/new-auto-scaling-for-emr-clusters\/\">New \u2013 Auto Scaling for EMR Clusters<\/a>\uc758 \ud55c\uad6d\uc5b4 \ubc88\uc5ed\uc785\ub2c8\ub2e4.<\/em><\/p>\n<p>Source: <a href=\"https:\/\/aws.amazon.com\/ko\/blogs\/korea\/new-auto-scaling-for-emr-clusters\/\" target=\"_blank\">Amazon EMR \ud074\ub7ec\uc2a4\ud130 \uc790\ub3d9 \ud655\uc7a5 \uae30\ub2a5 \ucd94\uac00<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<div class=\"mh-excerpt\"><p>Amazon EMR \ud074\ub7ec\uc2a4\ud130 \uc790\ub3d9 \ud655\uc7a5 \uae30\ub2a5 \ucd94\uac00 Amazon EMR \ud300\uc740 \ucd5c\uadfc \uc2e0\uaddc \ubc84\uc804\uc744 \uacc4\uc18d \ucd9c\uc2dc\ud558\uba74\uc11c, \uc774\ubc88 \ubd84\uae30\uc5d0\ub9cc \ub2e4\uc591\ud55c \uae30\ub2a5\uc744 \ucd94\uac00\ud558\uc600\uc2b5\ub2c8\ub2e4. 9\uc6d4 \u2013 Apache Spark, Tez, Hadoop MapReduce \ub370\uc774\ud130 \uc554\ud638\ud654. 9\uc6d4 \u2013 Apache Hive\ub97c \uc704\ud55c \uc624\ud508 \uc18c\uc2a4 EMR-DynamoDB Connector \uc9c0\uc6d0. 11\uc6d4 \u2013 Apache Flink\uc5d0\uc11c \uc2e4\uc2dc\uac04 \uc2a4\ud2b8\ub9ac\ubc0d \ucc98\ub9ac. 11\uc6d4 \u2013 \ud074\ub7ec\uc2a4\ud130 \ud0dc\uadf8\ub97c \ud65c\uc6a9\ud55c \uc138\ubd80 \uad00\ub9ac \uae30\ub2a5. \uc624\ub298\ubd80\ud130 Amazon EMR \ud074\ub7ec\uc2a4\ud130\uc5d0 \ub300\ud55c \uc790\ub3d9 \ud655\uc7a5 \uae30\ub2a5\uc744 \uc81c\uacf5\ud569\ub2c8\ub2e4. \uc774\uc81c \uc218\ud3c9 \ud655\uc7a5 \ubc0f \uac10\uc18c \uae30\ub2a5\uc744 \uc0ac\uc6a9\ud558\uc5ec \uc791\uc5c5\ub7c9 \ubcc0\uacbd\uc5d0 \ub300\ud55c \ud074\ub7ec\uc2a4\ud130\uc758 \ucf54\uc5b4 \ubc0f \uc791\uc5c5 \ub178\ub4dc \uc218\ub97c \uc870\uc815\ud558\uace0 \uc790\uc6d0 \uc0ac\uc6a9\uc744 \ucd5c\uc801\ud654 \ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc218\ud3c9 \ud655\uc7a5(Scale-out) \uc815\ucc45: \ucef4\ud4e8\ud305 \uc6a9\ub7c9\uc744 \ucd94\uac00\ud558\uc5ec \ub354 \ud070 \ubb38\uc81c\ub97c \ud574\uacb0\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. Apache Spark \ubc0f Apache Hive\uc640 \uac19\uc740 \uc751\uc6a9 \ud504\ub85c\uadf8\ub7a8\uc744 \uc790\ub3d9\uc73c\ub85c \uc99d\uac00\ud558\ub294 \ucc98\ub9ac \ub2a5\ub825\uc744 \ud65c\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc218\ud3c9 \uac10\uc18c(Scale-in) \uc815\ucc45: \uc778\uc2a4\ud134\uc2a4 \uc0ac\uc6a9 \uc2dc\uac04\uc774 \ub05d\ub098\uac70\ub098 \uc804\uccb4 \uc791\uc5c5\uc758 \ub85c\ub4dc\uac00 \uc904\uc5b4\ub4e4\ub54c \uc6a9\ub7c9\uc744 \uc81c\uac70\ud569\ub2c8\ub2e4. YARN \uad00\ub9ac \ucf58\ud14c\uc774\ub108\uc5d0\uc11c \ub178\ub4dc\uac00 \ud558\ub098 \uc904\uc5b4 \ub4e4\uba74, YARN\uc740 \ub2e4\ub978 \ub178\ub4dc\uc5d0\uc11c \uc7ac\uc2e4\ud589\ub429\ub2c8\ub2e4. (\uc790\uc138\ud55c \uc0ac\ud56d\uc740 Configure <a class=\"mh-excerpt-more\" href=\"https:\/\/jirak.net\/wp\/amazon-emr-%ed%81%b4%eb%9f%ac%ec%8a%a4%ed%84%b0-%ec%9e%90%eb%8f%99-%ed%99%95%ec%9e%a5-%ea%b8%b0%eb%8a%a5-%ec%b6%94%ea%b0%80\/\" title=\"Amazon EMR \ud074\ub7ec\uc2a4\ud130 \uc790\ub3d9 \ud655\uc7a5 \uae30\ub2a5 \ucd94\uac00\">[ more&#8230; ]<\/a><\/p>\n<\/div>","protected":false},"author":1,"featured_media":12047,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[169],"tags":[656],"class_list":["post-12046","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-news","tag-aws"],"amp_enabled":true,"_links":{"self":[{"href":"https:\/\/jirak.net\/wp\/wp-json\/wp\/v2\/posts\/12046","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/jirak.net\/wp\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/jirak.net\/wp\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/jirak.net\/wp\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/jirak.net\/wp\/wp-json\/wp\/v2\/comments?post=12046"}],"version-history":[{"count":1,"href":"https:\/\/jirak.net\/wp\/wp-json\/wp\/v2\/posts\/12046\/revisions"}],"predecessor-version":[{"id":12048,"href":"https:\/\/jirak.net\/wp\/wp-json\/wp\/v2\/posts\/12046\/revisions\/12048"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/jirak.net\/wp\/wp-json\/wp\/v2\/media\/12047"}],"wp:attachment":[{"href":"https:\/\/jirak.net\/wp\/wp-json\/wp\/v2\/media?parent=12046"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/jirak.net\/wp\/wp-json\/wp\/v2\/categories?post=12046"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/jirak.net\/wp\/wp-json\/wp\/v2\/tags?post=12046"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}