华为云首页 用户手册

MapReduce服务 MRS-创建集群并执行作业:请求示例

MapReduce服务 MRS-创建集群并执行作业:请求示例

时间：2023-11-01 16:25:08
MapReduce服务 MRS

请求示例

  • 使用node_groups参数组，创建一个启用“集群高可用”功能的集群，集群版本号为MRS 3.1.0。

    POST https://{endpoint}/v1.1/{project_id}/run-job-flow{  "billing_type" : 12,  "data_center" : "",  "available_zone_id" : "d573142f24894ef3bd3664de068b44b0",  "cluster_name" : "mrs_HEbK",  "cluster_version" : "MRS 3.1.0",  "safe_mode" : 0,  "cluster_type" : 0,  "component_list" : [ {    "component_name" : "Hadoop"  }, {    "component_name" : "Spark"  }, {    "component_name" : "HBase"  }, {    "component_name" : "Hive"  }, {    "component_name" : "Presto"  }, {    "component_name" : "Tez"  }, {    "component_name" : "Hue"  }, {    "component_name" : "Loader"  }, {    "component_name" : "Flink"  } ],  "vpc" : "vpc-4b1c",  "vpc_id" : "4a365717-67be-4f33-80c5-98e98a813af8",  "subnet_id" : "67984709-e15e-4e86-9886-d76712d4e00a",  "subnet_name" : "subnet-4b44",  "security_groups_id" : "4820eace-66ad-4f2c-8d46-cf340e3029dd",  "enterprise_project_id" : "0",  "tags" : [ {    "key" : "key1",    "value" : "value1"  }, {    "key" : "key2",    "value" : "value2"  } ],  "node_groups" : [ {    "group_name" : "master_node_default_group",    "node_num" : 2,    "node_size" : "s3.xlarge.2.linux.bigdata",    "root_volume_size" : 480,    "root_volume_type" : "SATA",    "data_volume_type" : "SATA",    "data_volume_count" : 1,    "data_volume_size" : 600  }, {    "group_name" : "core_node_analysis_group",    "node_num" : 3,    "node_size" : "s3.xlarge.2.linux.bigdata",    "root_volume_size" : 480,    "root_volume_type" : "SATA",    "data_volume_type" : "SATA",    "data_volume_count" : 1,    "data_volume_size" : 600  }, {    "group_name" : "task_node_analysis_group",    "node_num" : 2,    "node_size" : "s3.xlarge.2.linux.bigdata",    "root_volume_size" : 480,    "root_volume_type" : "SATA",    "data_volume_type" : "SATA",    "data_volume_count" : 0,    "data_volume_size" : 600,    "auto_scaling_policy" : {      "auto_scaling_enable" : true,      "min_capacity" : 1,      "max_capacity" : "3",      "resources_plans" : [ {        "period_type" : "daily",        "start_time" : "9:50",        "end_time" : "10:20",        "min_capacity" : 2,        "max_capacity" : 3      }, {        "period_type" : "daily",        "start_time" : "10:20",        "end_time" : "12:30",        "min_capacity" : 0,        "max_capacity" : 2      } ],      "exec_scripts" : [ {        "name" : "before_scale_out",        "uri" : "s3a://XXX/zeppelin_install.sh",        "parameters" : "${mrs_scale_node_num} ${mrs_scale_type} xxx",        "nodes" : [ "master", "core", "task" ],        "active_master" : "true",        "action_stage" : "before_scale_out",        "fail_action" : "continue"      }, {        "name" : "after_scale_out",        "uri" : "s3a://XXX/storm_rebalance.sh",        "parameters" : "${mrs_scale_node_hostnames} ${mrs_scale_node_ips}",        "nodes" : [ "master", "core", "task" ],        "active_master" : "true",        "action_stage" : "after_scale_out",        "fail_action" : "continue"      } ],      "rules" : [ {        "name" : "default-expand-1",        "adjustment_type" : "scale_out",        "cool_down_minutes" : 5,        "scaling_adjustment" : 1,        "trigger" : {          "metric_name" : "YARNMemoryAvailablePercentage",          "metric_value" : "25",          "comparison_operator" : "LT",          "evaluation_periods" : 10        }      }, {        "name" : "default-shrink-1",        "adjustment_type" : "scale_in",        "cool_down_minutes" : 5,        "scaling_adjustment" : 1,        "trigger" : {          "metric_name" : "YARNMemoryAvailablePercentage",          "metric_value" : "70",          "comparison_operator" : "GT",          "evaluation_periods" : 10        }      } ]    }  } ],  "login_mode" : 1,  "cluster_master_secret" : "",  "cluster_admin_secret" : "",  "log_collection" : 1,  "add_jobs" : [ {    "job_type" : 1,    "job_name" : "tenji111",    "jar_path" : "s3a://bigdata/program/hadoop-mapreduce-examples-2.7.2.jar",    "arguments" : "wordcount",    "input" : "s3a://bigdata/input/wd_1k/",    "output" : "s3a://bigdata/ouput/",    "job_log" : "s3a://bigdata/log/",    "shutdown_cluster" : true,    "file_action" : "",    "submit_job_once_cluster_run" : true,    "hql" : "",    "hive_script_path" : ""  } ],  "bootstrap_scripts" : [ {    "name" : "Modify os config",    "uri" : "s3a://XXX/modify_os_config.sh",    "parameters" : "param1 param2",    "nodes" : [ "master", "core", "task" ],    "active_master" : "false",    "before_component_start" : "true",    "start_time" : "1667892101",    "state" : "IN_PROGRESS",    "fail_action" : "continue",    "action_stages" : [ "BEFORE_COMPONENT_FIRST_START", "BEFORE_SCALE_IN" ]  }, {    "name" : "Install zepplin",    "uri" : "s3a://XXX/zeppelin_install.sh",    "parameters" : "",    "nodes" : [ "master" ],    "active_master" : "true",    "before_component_start" : "false",    "start_time" : "1667892101",    "state" : "IN_PROGRESS",    "fail_action" : "continue",    "action_stages" : [ "AFTER_SCALE_IN", "AFTER_SCALE_OUT" ]  } ]}

  • 不使用node_groups参数组，创建一个启用“集群高可用”功能的集群，集群版本号为MRS 3.1.0。

    POST https://{endpoint}/v1.1/{project_id}/run-job-flow{  "billing_type" : 12,  "data_center" : "",  "master_node_num" : 2,  "master_node_size" : "s3.2xlarge.2.linux.bigdata",  "core_node_num" : 3,  "core_node_size" : "s1.xlarge.linux.bigdata",  "available_zone_id" : "d573142f24894ef3bd3664de068b44b0",  "cluster_name" : "newcluster",  "vpc" : "vpc1",  "vpc_id" : "5b7db34d-3534-4a6e-ac94-023cd36aaf74",  "subnet_id" : "815bece0-fd22-4b65-8a6e-15788c99ee43",  "subnet_name" : "subnet",  "security_groups_id" : "845bece1-fd22-4b45-7a6e-14338c99ee43",  "tags" : [ {    "key" : "key1",    "value" : "value1"  }, {    "key" : "key2",    "value" : "value2"  } ],  "cluster_version" : "MRS 3.1.0",  "cluster_type" : 0,  "master_data_volume_type" : "SATA",  "master_data_volume_size" : 600,  "master_data_volume_count" : 1,  "core_data_volume_type" : "SATA",  "core_data_volume_size" : 600,  "core_data_volume_count" : 2,  "node_public_cert_name" : "SSHkey-bba1",  "safe_mode" : 0,  "log_collection" : 1,  "task_node_groups" : [ {    "node_num" : 2,    "node_size" : "s3.xlarge.2.linux.bigdata",    "data_volume_type" : "SATA",    "data_volume_count" : 1,    "data_volume_size" : 600,    "auto_scaling_policy" : {      "auto_scaling_enable" : true,      "min_capacity" : 1,      "max_capacity" : "3",      "resources_plans" : [ {        "period_type" : "daily",        "start_time" : "9: 50",        "end_time" : "10: 20",        "min_capacity" : 2,        "max_capacity" : 3      }, {        "period_type" : "daily",        "start_time" : "10: 20",        "end_time" : "12: 30",        "min_capacity" : 0,        "max_capacity" : 2      } ],      "exec_scripts" : [ {        "name" : "before_scale_out",        "uri" : "s3a: //XXX/zeppelin_install.sh",        "parameters" : "${mrs_scale_node_num}${mrs_scale_type}xxx",        "nodes" : [ "master", "core", "task" ],        "active_master" : "true",        "action_stage" : "before_scale_out",        "fail_action" : "continue"      }, {        "name" : "after_scale_out",        "uri" : "s3a: //XXX/storm_rebalance.sh",        "parameters" : "${mrs_scale_node_hostnames}${mrs_scale_node_ips}",        "nodes" : [ "master", "core", "task" ],        "active_master" : "true",        "action_stage" : "after_scale_out",        "fail_action" : "continue"      } ],      "rules" : [ {        "name" : "default-expand-1",        "adjustment_type" : "scale_out",        "cool_down_minutes" : 5,        "scaling_adjustment" : 1,        "trigger" : {          "metric_name" : "YARNMemoryAvailablePercentage",          "metric_value" : "25",          "comparison_operator" : "LT",          "evaluation_periods" : 10        }      }, {        "name" : "default-shrink-1",        "adjustment_type" : "scale_in",        "cool_down_minutes" : 5,        "scaling_adjustment" : 1,        "trigger" : {          "metric_name" : "YARNMemoryAvailablePercentage",          "metric_value" : "70",          "comparison_operator" : "GT",          "evaluation_periods" : 10        }      } ]    }  } ],  "component_list" : [ {    "component_name" : "Hadoop"  }, {    "component_name" : "Spark"  }, {    "component_name" : "HBase"  }, {    "component_name" : "Hive"  } ],  "add_jobs" : [ {    "job_type" : 1,    "job_name" : "tenji111",    "jar_path" : "s3a: //bigdata/program/hadoop-mapreduce-examples-2.7.2.jar",    "arguments" : "wordcount",    "input" : "s3a: //bigdata/input/wd_1k/",    "output" : "s3a: //bigdata/ouput/",    "job_log" : "s3a: //bigdata/log/",    "shutdown_cluster" : true,    "file_action" : "",    "submit_job_once_cluster_run" : true,    "hql" : "",    "hive_script_path" : ""  } ],  "bootstrap_scripts" : [ {    "name" : "Modifyosconfig",    "uri" : "s3a: //XXX/modify_os_config.sh",    "parameters" : "param1param2",    "nodes" : [ "master", "core", "task" ],    "active_master" : "false",    "before_component_start" : "true",    "start_time" : "1667892101",    "state" : "IN_PROGRESS",    "fail_action" : "continue",    "action_stages" : [ "BEFORE_COMPONENT_FIRST_START", "BEFORE_SCALE_IN" ]  }, {    "name" : "Installzepplin",    "uri" : "s3a: //XXX/zeppelin_install.sh",    "parameters" : "",    "nodes" : [ "master" ],    "active_master" : "true",    "before_component_start" : "false",    "start_time" : "1667892101",    "state" : "IN_PROGRESS",    "fail_action" : "continue",    "action_stages" : [ "AFTER_SCALE_IN", "AFTER_SCALE_OUT" ]  } ]}

  • 使用node_groups参数组，创建一个关闭“集群高可用”功能、最小规格的集群，集群版本号为MRS 3.1.0。

    POST https://{endpoint}/v1.1/{project_id}/run-job-flow{  "billing_type" : 12,  "data_center" : "",  "available_zone_id" : "d573142f24894ef3bd3664de068b44b0",  "cluster_name" : "mrs_HEbK",  "cluster_version" : "MRS 3.1.0",  "safe_mode" : 0,  "cluster_type" : 0,  "component_list" : [ {    "component_name" : "Hadoop"  }, {    "component_name" : "Spark"  }, {    "component_name" : "HBase"  }, {    "component_name" : "Hive"  }, {    "component_name" : "Presto"  }, {    "component_name" : "Tez"  }, {    "component_name" : "Hue"  }, {    "component_name" : "Loader"  }, {    "component_name" : "Flink"  } ],  "vpc" : "vpc-4b1c",  "vpc_id" : "4a365717-67be-4f33-80c5-98e98a813af8",  "subnet_id" : "67984709-e15e-4e86-9886-d76712d4e00a",  "subnet_name" : "subnet-4b44",  "security_groups_id" : "4820eace-66ad-4f2c-8d46-cf340e3029dd",  "enterprise_project_id" : "0",  "tags" : [ {    "key" : "key1",    "value" : "value1"  }, {    "key" : "key2",    "value" : "value2"  } ],  "node_groups" : [ {    "group_name" : "master_node_default_group",    "node_num" : 1,    "node_size" : "s3.xlarge.2.linux.bigdata",    "root_volume_size" : 480,    "root_volume_type" : "SATA",    "data_volume_type" : "SATA",    "data_volume_count" : 1,    "data_volume_size" : 600  }, {    "group_name" : "core_node_analysis_group",    "node_num" : 1,    "node_size" : "s3.xlarge.2.linux.bigdata",    "root_volume_size" : 480,    "root_volume_type" : "SATA",    "data_volume_type" : "SATA",    "data_volume_count" : 1,    "data_volume_size" : 600  } ],  "login_mode" : 1,  "cluster_master_secret" : "",  "cluster_admin_secret" : "",  "log_collection" : 1,  "add_jobs" : [ {    "job_type" : 1,    "job_name" : "tenji111",    "jar_path" : "s3a://bigdata/program/hadoop-mapreduce-examples-2.7.2.jar",    "arguments" : "wordcount",    "input" : "s3a://bigdata/input/wd_1k/",    "output" : "s3a://bigdata/ouput/",    "job_log" : "s3a://bigdata/log/",    "shutdown_cluster" : true,    "file_action" : "",    "submit_job_once_cluster_run" : true,    "hql" : "",    "hive_script_path" : ""  } ],  "bootstrap_scripts" : [ {    "name" : "Modify os config",    "uri" : "s3a://XXX/modify_os_config.sh",    "parameters" : "param1 param2",    "nodes" : [ "master", "core", "task" ],    "active_master" : "false",    "before_component_start" : "true",    "start_time" : "1667892101",    "state" : "IN_PROGRESS",    "fail_action" : "continue",    "action_stages" : [ "BEFORE_COMPONENT_FIRST_START", "BEFORE_SCALE_IN" ]  }, {    "name" : "Install zepplin",    "uri" : "s3a://XXX/zeppelin_install.sh",    "parameters" : "",    "nodes" : [ "master" ],    "active_master" : "true",    "before_component_start" : "false",    "start_time" : "1667892101",    "state" : "IN_PROGRESS",    "fail_action" : "continue",    "action_stages" : [ "AFTER_SCALE_IN", "AFTER_SCALE_OUT" ]  } ]}

  • 不使用node_groups参数组，创建一个关闭“集群高可用”功能、最小规格的集群，集群版本号为MRS 3.1.0。

    POST https://{endpoint}/v1.1/{project_id}/run-job-flow{  "billing_type" : 12,  "data_center" : "",  "master_node_num" : 1,  "master_node_size" : "s3.2xlarge.2.linux.bigdata",  "core_node_num" : 1,  "core_node_size" : "s1.xlarge.linux.bigdata",  "available_zone_id" : "d573142f24894ef3bd3664de068b44b0",  "cluster_name" : "newcluster",  "vpc" : "vpc1",  "vpc_id" : "5b7db34d-3534-4a6e-ac94-023cd36aaf74",  "subnet_id" : "815bece0-fd22-4b65-8a6e-15788c99ee43",  "subnet_name" : "subnet",  "security_groups_id" : "",  "enterprise_project_id" : "0",  "tags" : [ {    "key" : "key1",    "value" : "value1"  }, {    "key" : "key2",    "value" : "value2"  } ],  "cluster_version" : "MRS 3.1.0",  "cluster_type" : 0,  "master_data_volume_type" : "SATA",  "master_data_volume_size" : 600,  "master_data_volume_count" : 1,  "core_data_volume_type" : "SATA",  "core_data_volume_size" : 600,  "core_data_volume_count" : 1,  "login_mode" : 1,  "node_public_cert_name" : "SSHkey-bba1",  "safe_mode" : 0,  "cluster_admin_secret" : "******",  "log_collection" : 1,  "component_list" : [ {    "component_name" : "Hadoop"  }, {    "component_name" : "Spark"  }, {    "component_name" : "HBase"  }, {    "component_name" : "Hive"  }, {    "component_name" : "Presto"  }, {    "component_name" : "Tez"  }, {    "component_name" : "Hue"  }, {    "component_name" : "Loader"  }, {    "component_name" : "Flink"  } ],  "add_jobs" : [ {    "job_type" : 1,    "job_name" : "tenji111",    "jar_path" : "s3a://bigdata/program/hadoop-mapreduce-examples-XXX.jar",    "arguments" : "wordcount",    "input" : "s3a://bigdata/input/wd_1k/",    "output" : "s3a://bigdata/ouput/",    "job_log" : "s3a://bigdata/log/",    "shutdown_cluster" : false,    "file_action" : "",    "submit_job_once_cluster_run" : true,    "hql" : "",    "hive_script_path" : ""  } ],  "bootstrap_scripts" : [ {    "name" : "Install zepplin",    "uri" : "s3a://XXX/zeppelin_install.sh",    "parameters" : "",    "nodes" : [ "master" ],    "active_master" : "false",    "before_component_start" : "false",    "start_time" : "1667892101",    "state" : "IN_PROGRESS",    "fail_action" : "continue",    "action_stages" : [ "AFTER_SCALE_IN", "AFTER_SCALE_OUT" ]  } ]}
support.huaweicloud.com/api-mrs/mrs_02_0028.html
看了此文的人还看了
CDN加速 GaussDB 文字转换成语音 免费的服务器 如何创建网站 域名网站购买 私有云桌面 云主机哪个好 域名怎么备案 手机云电脑 SSL证书申请 云点播服务器 免费OCR是什么 电脑云桌面 域名备案怎么弄 语音转文字 文字图片识别 云桌面是什么 网址安全检测 网站建设搭建 国外CDN加速 SSL免费证书申请 短信批量发送 图片OCR识别 云数据库MySQL 个人域名购买 录音转文字 扫描图片识别文字 OCR图片识别 行驶证识别 虚拟电话号码 电话呼叫中心软件 怎么制作一个网站 Email注册网站 华为VNC 图像文字识别 企业网站制作 个人网站搭建 华为云计算 免费租用云托管 云桌面云服务器 ocr文字识别免费版 HTTPS证书申请 图片文字识别转换 国外域名注册商 使用免费虚拟主机 云电脑主机多少钱 鲲鹏云手机 短信验证码平台 OCR图片文字识别 SSL证书是什么 申请企业邮箱步骤 免费的企业用邮箱 云免流搭建教程 域名价格
MapReduce服务 MRS-创建集群并执行作业:请求示例

搜索反馈

您搜索到想要的结果了吗?

是的 没有

意见反馈

0/200

提交 取消

提交成功！非常感谢您的反馈，我们会继续努力做到更好 反馈提交失败！请稍后重试！
热门活动
推荐文章
更多内容
云硬盘存储EVS Web应用防火墙 人机语音识别 开发者大会2023 云计算入门 云数据库选择 HDC大会 华为云计算 响应式建站是什么 智慧班牌系统