May
23

Nutanix Calm Blueprint for Auto Scaling Compute Resources for Cloudera

Adding additional compute resources by first looking at seasonal trending data from Prism Central. Auto scaling is done with using Nutanix Calm.

Save the below code as a json file.
cdh_parcel.version:\n cdh_parcel = p\n\nprint cdh_parcel.product + ' ' + cdh_parcel.version + \" downloaded\"\n\n# distribute the parcel\nprint \"Starting parcel distribution. This might take a while.\"\ncmd = cdh_parcel.start_distribution()\nif cmd.success != True:\n print \"Parcel distribution failed!\"\n exit(0)\n\nprint 'wait 4 min'\n# make sure the distribution finishes\n#while cdh_parcel.stage != \"DISTRIBUTED\":\nsleep(200)\n# cdh_parcel = get_parcel(api, cdh_parcel.product, '5.13.3-1.cdh5.13.3.p0.2', cluster_name)\n\nprint cdh_parcel.product + ' ' + cdh_parcel.version + \" distributed\"\n\n# activate the parcel\ncmd = cdh_parcel.activate()\nif cmd.success != True:\n print \"Parcel activation failed!\"\n exit(0)\n\n# make sure the activation finishes\n#while cdh_parcel.stage != \"ACTIVATED\":\n# cdh_parcel = get_parcel(api, cdh_parcel.product, '5.13.3-1.cdh5.13.3.p0.2', cluster_name)\n\nprint cdh_parcel.product + ' ' + cdh_parcel.version + \" activated\"\n\n\nyarn = cluster.get_service('yarn')\nprint yarn.name\nhostsid = api.get_all_hosts()\n#range = len(hostsid)\n#print 'range: ' + range\n\nfor current in hostsid:\n if current.rackId == '/default/default':\n print 'host name to add' + current.hostname\n currentshort1 = current.hostname\n currentshort = currentshort1.split(\".\", 1)\n short = currentshort[0]\n print short\n yarn.create_role(short, 'NODEMANAGER', current.hostId)\n\ncluster.restart(restart_only_stale_services=True, redeploy_client_configuration=True)\n\nexit(0)", "script_type": "sh", "type": "", "command_line_args": "", "login_credential_local_reference": {"kind": "app_credential", "name": "go", "uuid": "f356d2b6-2fac-453c-84ad-d91bc21bb1ba"}}, "timeout_secs": "", "type": "EXEC", "variable_list": [], "uuid": "b24dd948-e65d-2c1e-52d8-c4d21d615274"}], "description": "", "name": "7da2879a_runbook_cloned_1", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "6405be68_dag", "uuid": "7ec268af-4e53-3c55-930a-740501d41cfb"}, "message_list": [], "variable_list": [], "uuid": "2363d280-af23-b97a-557f-1e6a3d156fbb"}, "type": "", "uninstall_runbook": {"task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "de4520ca_dag", "state": "ACTIVE", "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "uuid": "aabae04b-08bb-466e-76a2-1fcfc3c816cd"}], "description": "", "name": "339b443d_runbook_cloned_1", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "de4520ca_dag", "uuid": "aabae04b-08bb-466e-76a2-1fcfc3c816cd"}, "message_list": [], "variable_list": [], "uuid": "ff185ae5-bc26-095b-6840-9fc957638621"}}, "timeout_secs": "", "type": "CUSTOM_PACKAGE_INSTALL", "variable_list": [], "uuid": "47831e03-3fa0-4a8e-b26c-8244da30ab82"}], "description": "", "name": "SYS_GEN__Runbook_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "SYS_GEN__DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "078dca59-9b50-4ad8-b048-2ca148f8251e"}, "message_list": [], "variable_list": [], "uuid": "b81bf28e-424c-4daa-9beb-8ae9a62855ba"}, "message_list": [], "name": "action_install"}, {"description": "System action for creating an application", "type": "system", "uuid": "563a264b-ca8e-4279-ad98-96e4e8d77282", "state": "ACTIVE", "critical": true, "attrs": {}, "runbook": {"task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [{"kind": "app_task", "name": "SYS_GEN__install_CRb_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "d4857a2b-32f6-471d-9e05-295df9e45c87"}, {"kind": "app_task", "name": "SYS_GEN__create_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "55697484-6734-4bdd-afde-75e8c171697e"}, {"kind": "app_task", "name": "SYS_GEN__start_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "d5751cd2-f72a-4f35-9fd6-104079471e54"}], "name": "SYS_GEN__Composite_DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "attrs": {"edges": [{"from_task_reference": {"kind": "app_task", "name": "SYS_GEN__create_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "55697484-6734-4bdd-afde-75e8c171697e"}, "edge_type": "inherent", "type": "", "uuid": "873c853d-d849-46a7-9344-625a6b9c0898", "to_task_reference": {"kind": "app_task", "name": "SYS_GEN__start_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "d5751cd2-f72a-4f35-9fd6-104079471e54"}}, {"from_task_reference": {"kind": "app_task", "name": "SYS_GEN__install_CRb_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "d4857a2b-32f6-471d-9e05-295df9e45c87"}, "edge_type": "inherent", "type": "", "uuid": "d316c8f7-435c-4b1c-ad0c-7b1d02696dcd", "to_task_reference": {"kind": "app_task", "name": "SYS_GEN__create_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "55697484-6734-4bdd-afde-75e8c171697e"}}], "type": "DAG"}, "timeout_secs": "", "type": "DAG", "variable_list": [], "uuid": "dcc6c9a0-ef16-406c-8960-b313800452b3"}, {"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "SYS_GEN__install_CRb_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "attrs": {"type": "CALL_RUNBOOK", "inarg_list": [], "runbook_reference": {"kind": "app_runbook", "name": "SYS_GEN__Runbook_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "b81bf28e-424c-4daa-9beb-8ae9a62855ba"}}, "timeout_secs": "", "type": "CALL_RUNBOOK", "variable_list": [], "uuid": "d4857a2b-32f6-471d-9e05-295df9e45c87"}, {"target_any_local_reference": {"kind": "app_service", "name": "code", "uuid": "0afc9748-88f7-8e8c-f7fe-b2b2cd6669e1"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "SYS_GEN__create_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "state": "ACTIVE", "attrs": {"type": "CALL_RUNBOOK", "inarg_list": [], "runbook_reference": {"kind": "app_runbook", "name": "eba78c76_runbook_cloned_1", "uuid": "e3ec95e5-6ede-c168-b629-600d771f5803"}}, "timeout_secs": "", "type": "CALL_RUNBOOK", "variable_list": [], "uuid": "55697484-6734-4bdd-afde-75e8c171697e"}, {"target_any_local_reference": {"kind": "app_service", "name": "code", "uuid": "0afc9748-88f7-8e8c-f7fe-b2b2cd6669e1"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "SYS_GEN__start_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "state": "ACTIVE", "attrs": {"type": "CALL_RUNBOOK", "inarg_list": [], "runbook_reference": {"kind": "app_runbook", "name": "3cedb317_runbook_cloned_1", "uuid": "aeedc907-4495-18e4-e7aa-93b781b90737"}}, "timeout_secs": "", "type": "CALL_RUNBOOK", "variable_list": [], "uuid": "d5751cd2-f72a-4f35-9fd6-104079471e54"}], "description": "", "name": "SYS_GEN__Runbook_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "SYS_GEN__Composite_DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "dcc6c9a0-ef16-406c-8960-b313800452b3"}, "message_list": [], "variable_list": [], "uuid": "daf9342b-98d9-4877-aee9-b322ed207462"}, "message_list": [], "name": "action_create"}, {"description": "System action for uninstalling an application", "type": "system", "uuid": "3255d19b-f2af-4d1f-a517-f91bf14bdc93", "state": "ACTIVE", "critical": true, "attrs": {}, "runbook": {"task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [{"kind": "app_task", "name": "PE_Uninstall_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "aa9bdd7f-4d40-4d9c-af4d-13091c066f70"}], "name": "SYS_GEN__DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "attrs": {"edges": [], "type": "DAG"}, "timeout_secs": "", "type": "DAG", "variable_list": [], "uuid": "92cd3b4b-fc1a-4dc3-9f17-6a5de9a2473e"}, {"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "PE_Uninstall_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "attrs": {"install_runbook": {"task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [{"kind": "app_task", "name": "PackageInstallTask", "uuid": "b24dd948-e65d-2c1e-52d8-c4d21d615274"}], "name": "6405be68_dag", "state": "ACTIVE", "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "uuid": "7ec268af-4e53-3c55-930a-740501d41cfb"}, {"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "PackageInstallTask", "state": "ACTIVE", "attrs": {"exit_status": [], "script": "#!/usr/bin/env python\nfrom cm_api.api_client import ApiResource\nfrom cm_api.endpoints.clusters import ApiCluster\nfrom cm_api.endpoints.clusters import create_cluster\nfrom cm_api.endpoints.parcels import ApiParcel\nfrom cm_api.endpoints.parcels import get_parcel\nfrom cm_api.endpoints.cms import ClouderaManager\nfrom cm_api.endpoints.services import ApiService, ApiServiceSetupInfo\nfrom cm_api.endpoints.services import create_service\nfrom cm_api.endpoints.types import ApiCommand, ApiRoleConfigGroupRef\nfrom cm_api.endpoints.role_config_groups import get_role_config_group\nfrom cm_api.endpoints.role_config_groups import ApiRoleConfigGroup\nfrom cm_api.endpoints.roles import ApiRole\nfrom time import sleep\n\nhost_username = 'root'\ncm_repo_url = None\ncdh_version = \"CDH5\" # also valid: \"CDH4\"\ncdh_version_number = \"5\"\n#240\nsleep(240)\n\nhostsIP = [ ]\nhostNames = [ ]\nhostNamesLong = [ ]\nthostNames = [ ]\nthostNames = \"@@{compute.name}@@\"\n#hostsIP = thostsIP.split(,)\nhostNames = thostNames.split(\",\")\n\n#for name in hostName:\n # hostNamesLong = hostName.append(name + '.rf9.local')\n\nfor name in hostNames:\n hostNamesLong.append(name + '.rf9.local')\n print hostNamesLong\n \nprint \"@@{code.cmip}@@\"\nprint \"@@{code.cmPassword}@@\"\n\napi = ApiResource(\"@@{code.cmip}@@\", '7180', \"@@{code.cmAdmin}@@\", \"@@{code.cmPassword}@@\", version=\"@@{code.APIversion}@@\")\n\n#get the CM instance\ncm = api.get_cluster(\"@@{code.clusterName}@@\")\ncm = ClouderaManager(api)\n\n\ncmd = cm.host_install(\"@@{code.hostAdmin}@@\", hostNamesLong, password=\"@@{code.hostPassword}@@\", cm_repo_url=cm_repo_url)\nprint \"Installing hosts. This might take a while.\"\nwhile cmd.success == None:\n sleep(15)\n cmd = cmd.fetch()\n\nif cmd.success != True:\n print \"cm_host_install failed: \" + cmd.resultMessage\n# exit(0)\nprint \"cm_host_install succeeded\"\n\ncluster = api.get_cluster(\"@@{code.clusterName}@@\")\n\nhostsadded = cluster.add_hosts(hostNamesLong)\n\nparcels_list = []\n# get and list all available parcels\nprint \"Available parcels:\"\nfor p in cluster.get_all_parcels():\n print p.version\n if p.version.startswith(\"@@{code.cmPackage}@@\") and p.product == \"CDH\":\n parcels_list.append(p)\n\nif len(parcels_list) == 0:\n print \"No \"\n print cdh_version\n print \" parcel found!\"\n exit(0)\n\ncdh_parcel = parcels_list[0]\n\nfor p in parcels_list:\n if p.version > cdh_parcel.version:\n cdh_parcel = p\n\nprint cdh_parcel.product + ' ' + cdh_parcel.version + \" downloaded\"\n\n# distribute the parcel\nprint \"Starting parcel distribution. This might take a while.\"\ncmd = cdh_parcel.start_distribution()\nif cmd.success != True:\n print \"Parcel distribution failed!\"\n exit(0)\n\nprint 'wait 4 min'\n# make sure the distribution finishes\n#while cdh_parcel.stage != \"DISTRIBUTED\":\nsleep(200)\n# cdh_parcel = get_parcel(api, cdh_parcel.product, '5.13.3-1.cdh5.13.3.p0.2', cluster_name)\n\nprint cdh_parcel.product + ' ' + cdh_parcel.version + \" distributed\"\n\n# activate the parcel\ncmd = cdh_parcel.activate()\nif cmd.success != True:\n print \"Parcel activation failed!\"\n exit(0)\n\n# make sure the activation finishes\n#while cdh_parcel.stage != \"ACTIVATED\":\n# cdh_parcel = get_parcel(api, cdh_parcel.product, '5.13.3-1.cdh5.13.3.p0.2', cluster_name)\n\nprint cdh_parcel.product + ' ' + cdh_parcel.version + \" activated\"\n\n\nyarn = cluster.get_service('yarn')\nprint yarn.name\nhostsid = api.get_all_hosts()\n#range = len(hostsid)\n#print 'range: ' + range\n\nfor current in hostsid:\n if current.rackId == '/default/default':\n print 'host name to add' + current.hostname\n currentshort1 = current.hostname\n currentshort = currentshort1.split(\".\", 1)\n short = currentshort[0]\n print short\n yarn.create_role(short, 'NODEMANAGER', current.hostId)\n\ncluster.restart(restart_only_stale_services=True, redeploy_client_configuration=True)\n\nexit(0)", "script_type": "sh", "type": "", "command_line_args": "", "login_credential_local_reference": {"kind": "app_credential", "name": "go", "uuid": "f356d2b6-2fac-453c-84ad-d91bc21bb1ba"}}, "timeout_secs": "", "type": "EXEC", "variable_list": [], "uuid": "b24dd948-e65d-2c1e-52d8-c4d21d615274"}], "description": "", "name": "7da2879a_runbook_cloned_1", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "6405be68_dag", "uuid": "7ec268af-4e53-3c55-930a-740501d41cfb"}, "message_list": [], "variable_list": [], "uuid": "2363d280-af23-b97a-557f-1e6a3d156fbb"}, "type": "", "uninstall_runbook": {"task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "de4520ca_dag", "state": "ACTIVE", "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "uuid": "aabae04b-08bb-466e-76a2-1fcfc3c816cd"}], "description": "", "name": "339b443d_runbook_cloned_1", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "de4520ca_dag", "uuid": "aabae04b-08bb-466e-76a2-1fcfc3c816cd"}, "message_list": [], "variable_list": [], "uuid": "ff185ae5-bc26-095b-6840-9fc957638621"}}, "timeout_secs": "", "type": "CUSTOM_PACKAGE_UNINSTALL", "variable_list": [], "uuid": "aa9bdd7f-4d40-4d9c-af4d-13091c066f70"}], "description": "", "name": "SYS_GEN__Runbook_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "SYS_GEN__DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "92cd3b4b-fc1a-4dc3-9f17-6a5de9a2473e"}, "message_list": [], "variable_list": [], "uuid": "3efda7af-c2c5-4527-91b8-0c58ae037827"}, "message_list": [], "name": "action_uninstall"}, {"description": "System action for deleting an application. Deletes physical machines as well", "type": "system", "uuid": "7ad1b2bf-b87c-4d3d-8da5-33abeb3cb757", "state": "ACTIVE", "critical": true, "attrs": {}, "runbook": {"task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [{"kind": "app_task", "name": "SYS_GEN__stop_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "5b2b1a62-4e83-4d1b-934b-862d89fc58f9"}, {"kind": "app_task", "name": "SYS_GEN__delete_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "2ce000b8-1296-4f5d-a8ae-8da96433a1cf"}, {"kind": "app_task", "name": "SYS_GEN__uninstall_CRb_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "1a05ced5-575f-4daf-ae7c-033403a0e77d"}], "name": "SYS_GEN__Composite_DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "attrs": {"edges": [{"from_task_reference": {"kind": "app_task", "name": "SYS_GEN__stop_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "5b2b1a62-4e83-4d1b-934b-862d89fc58f9"}, "edge_type": "inherent", "type": "", "uuid": "60c3c99d-672f-49af-b994-89bd9f92909e", "to_task_reference": {"kind": "app_task", "name": "SYS_GEN__delete_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "2ce000b8-1296-4f5d-a8ae-8da96433a1cf"}}, {"from_task_reference": {"kind": "app_task", "name": "SYS_GEN__delete_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "2ce000b8-1296-4f5d-a8ae-8da96433a1cf"}, "edge_type": "inherent", "type": "", "uuid": "be945022-5672-4d55-9a03-54584bb4b48f", "to_task_reference": {"kind": "app_task", "name": "SYS_GEN__uninstall_CRb_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "1a05ced5-575f-4daf-ae7c-033403a0e77d"}}], "type": "DAG"}, "timeout_secs": "", "type": "DAG", "variable_list": [], "uuid": "24022115-efdb-4ecf-b9b2-6275a9944739"}, {"target_any_local_reference": {"kind": "app_service", "name": "code", "uuid": "0afc9748-88f7-8e8c-f7fe-b2b2cd6669e1"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "SYS_GEN__stop_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "state": "ACTIVE", "attrs": {"type": "CALL_RUNBOOK", "inarg_list": [], "runbook_reference": {"kind": "app_runbook", "name": "5be55dea_runbook_cloned_1", "uuid": "aec97efe-ee30-0743-f90f-192698db865c"}}, "timeout_secs": "", "type": "CALL_RUNBOOK", "variable_list": [], "uuid": "5b2b1a62-4e83-4d1b-934b-862d89fc58f9"}, {"target_any_local_reference": {"kind": "app_service", "name": "code", "uuid": "0afc9748-88f7-8e8c-f7fe-b2b2cd6669e1"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "SYS_GEN__delete_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "state": "ACTIVE", "attrs": {"type": "CALL_RUNBOOK", "inarg_list": [], "runbook_reference": {"kind": "app_runbook", "name": "740b4861_runbook_cloned_1", "uuid": "3b9eb309-30d3-a2ae-137b-9d1552ea6038"}}, "timeout_secs": "", "type": "CALL_RUNBOOK", "variable_list": [], "uuid": "2ce000b8-1296-4f5d-a8ae-8da96433a1cf"}, {"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "SYS_GEN__uninstall_CRb_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "attrs": {"type": "CALL_RUNBOOK", "inarg_list": [], "runbook_reference": {"kind": "app_runbook", "name": "SYS_GEN__Runbook_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "3efda7af-c2c5-4527-91b8-0c58ae037827"}}, "timeout_secs": "", "type": "CALL_RUNBOOK", "variable_list": [], "uuid": "1a05ced5-575f-4daf-ae7c-033403a0e77d"}], "description": "", "name": "SYS_GEN__Runbook_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "SYS_GEN__Composite_DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "24022115-efdb-4ecf-b9b2-6275a9944739"}, "message_list": [], "variable_list": [], "uuid": "13878dbf-d8a6-46ae-b2ba-91b2d94c80b6"}, "message_list": [], "name": "action_delete"}, {"description": "System action for deleting an application. Does not delete physical machines", "type": "system", "uuid": "514bcd42-76cc-4588-aaca-d8c090d2b394", "state": "ACTIVE", "critical": true, "attrs": {}, "runbook": {"task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [{"kind": "app_task", "name": "SYS_GEN__soft_delete_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "4378066f-61a2-455f-bac8-42fded9ad756"}, {"kind": "app_task", "name": "SYS_GEN__Package_Element_Delete_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "c3dcc740-7e1c-40b9-9c5f-80243f049090"}], "name": "SYS_GEN__Composite_DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "attrs": {"edges": [{"from_task_reference": {"kind": "app_task", "name": "SYS_GEN__soft_delete_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "4378066f-61a2-455f-bac8-42fded9ad756"}, "edge_type": "inherent", "type": "", "uuid": "eaa03831-dedd-430b-a855-b2f31ce3a575", "to_task_reference": {"kind": "app_task", "name": "SYS_GEN__Package_Element_Delete_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "c3dcc740-7e1c-40b9-9c5f-80243f049090"}}], "type": "DAG"}, "timeout_secs": "", "type": "DAG", "variable_list": [], "uuid": "b0c7ab87-e166-4b64-8e34-ecff50898741"}, {"target_any_local_reference": {"kind": "app_service", "name": "code", "uuid": "0afc9748-88f7-8e8c-f7fe-b2b2cd6669e1"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "SYS_GEN__soft_delete_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "state": "ACTIVE", "attrs": {"type": "CALL_RUNBOOK", "inarg_list": [], "runbook_reference": {"kind": "app_runbook", "name": "SYS_GEN__Runbook_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "d9c0f039-0740-41ee-9d53-d5846fb8aa33"}}, "timeout_secs": "", "type": "CALL_RUNBOOK", "variable_list": [], "uuid": "4378066f-61a2-455f-bac8-42fded9ad756"}, {"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "SYS_GEN__Package_Element_Delete_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "attrs": {"type": ""}, "timeout_secs": "", "type": "SOFT_DELETE_ELEMENT", "variable_list": [], "uuid": "c3dcc740-7e1c-40b9-9c5f-80243f049090"}], "description": "", "name": "SYS_GEN__Runbook_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "SYS_GEN__Composite_DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "b0c7ab87-e166-4b64-8e34-ecff50898741"}, "message_list": [], "variable_list": [], "uuid": "a33c5528-ee5d-45c1-a762-1130af869bfd"}, "message_list": [], "name": "action_soft_delete"}, {"description": "System action for starting an application", "type": "system", "uuid": "a85a6591-bc55-4b22-9057-dc49a156d6d0", "state": "ACTIVE", "critical": true, "attrs": {}, "runbook": {"task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [{"kind": "app_task", "name": "SYS_GEN__start_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "4fbf8a01-298d-4363-8968-6f4b034ee440"}], "name": "SYS_GEN__Composite_DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "attrs": {"edges": [], "type": "DAG"}, "timeout_secs": "", "type": "DAG", "variable_list": [], "uuid": "c340dcbc-2937-40b0-82a9-e82c9ea49f06"}, {"target_any_local_reference": {"kind": "app_service", "name": "code", "uuid": "0afc9748-88f7-8e8c-f7fe-b2b2cd6669e1"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "SYS_GEN__start_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "state": "ACTIVE", "attrs": {"type": "CALL_RUNBOOK", "inarg_list": [], "runbook_reference": {"kind": "app_runbook", "name": "3cedb317_runbook_cloned_1", "uuid": "aeedc907-4495-18e4-e7aa-93b781b90737"}}, "timeout_secs": "", "type": "CALL_RUNBOOK", "variable_list": [], "uuid": "4fbf8a01-298d-4363-8968-6f4b034ee440"}], "description": "", "name": "SYS_GEN__Runbook_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "SYS_GEN__Composite_DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "c340dcbc-2937-40b0-82a9-e82c9ea49f06"}, "message_list": [], "variable_list": [], "uuid": "12813311-3bf3-486f-b59a-ec1087c2f54d"}, "message_list": [], "name": "action_start"}, {"description": "System action for stopping an application", "type": "system", "uuid": "cf8d0527-c638-4d40-8947-a8bf1cb1d051", "state": "ACTIVE", "critical": true, "attrs": {}, "runbook": {"task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [{"kind": "app_task", "name": "SYS_GEN__stop_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "uuid": "9626db53-15f7-4f28-83b0-70061870f14f"}], "name": "SYS_GEN__Composite_DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "attrs": {"edges": [], "type": "DAG"}, "timeout_secs": "", "type": "DAG", "variable_list": [], "uuid": "fd0d81b5-563f-46bc-b001-b3e967bf45a0"}, {"target_any_local_reference": {"kind": "app_service", "name": "code", "uuid": "0afc9748-88f7-8e8c-f7fe-b2b2cd6669e1"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "SYS_GEN__stop_CRb_Service_0afc9748_88f7_8e8c_f7fe_b2b2cd6669e1", "state": "ACTIVE", "attrs": {"type": "CALL_RUNBOOK", "inarg_list": [], "runbook_reference": {"kind": "app_runbook", "name": "5be55dea_runbook_cloned_1", "uuid": "aec97efe-ee30-0743-f90f-192698db865c"}}, "timeout_secs": "", "type": "CALL_RUNBOOK", "variable_list": [], "uuid": "9626db53-15f7-4f28-83b0-70061870f14f"}], "description": "", "name": "SYS_GEN__Runbook_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "SYS_GEN__Composite_DAG_Package_44bbdab6_b71e_8989_d19b_aeb8c940122c", "uuid": "fd0d81b5-563f-46bc-b001-b3e967bf45a0"}, "message_list": [], "variable_list": [], "uuid": "6715e54e-21f8-4206-9d27-bb4518c0a012"}, "message_list": [], "name": "action_stop"}], "type": "DEB", "service_local_reference_list": [{"kind": "app_service", "name": "code", "uuid": "0afc9748-88f7-8e8c-f7fe-b2b2cd6669e1"}], "name": "cloudera", "state": "ACTIVE", "version": "", "editables": {}, "message_list": [], "options": {"install_runbook": {"task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [{"kind": "app_task", "name": "PackageInstallTask", "uuid": "b24dd948-e65d-2c1e-52d8-c4d21d615274"}], "name": "6405be68_dag", "state": "ACTIVE", "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "uuid": "7ec268af-4e53-3c55-930a-740501d41cfb"}, {"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "PackageInstallTask", "state": "ACTIVE", "attrs": {"exit_status": [], "script": "#!/usr/bin/env python\nfrom cm_api.api_client import ApiResource\nfrom cm_api.endpoints.clusters import ApiCluster\nfrom cm_api.endpoints.clusters import create_cluster\nfrom cm_api.endpoints.parcels import ApiParcel\nfrom cm_api.endpoints.parcels import get_parcel\nfrom cm_api.endpoints.cms import ClouderaManager\nfrom cm_api.endpoints.services import ApiService, ApiServiceSetupInfo\nfrom cm_api.endpoints.services import create_service\nfrom cm_api.endpoints.types import ApiCommand, ApiRoleConfigGroupRef\nfrom cm_api.endpoints.role_config_groups import get_role_config_group\nfrom cm_api.endpoints.role_config_groups import ApiRoleConfigGroup\nfrom cm_api.endpoints.roles import ApiRole\nfrom time import sleep\n\nhost_username = 'root'\ncm_repo_url = None\ncdh_version = \"CDH5\" # also valid: \"CDH4\"\ncdh_version_number = \"5\"\n#240\nsleep(240)\n\nhostsIP = [ ]\nhostNames = [ ]\nhostNamesLong = [ ]\nthostNames = [ ]\nthostNames = \"@@{compute.name}@@\"\n#hostsIP = thostsIP.split(,)\nhostNames = thostNames.split(\",\")\n\n#for name in hostName:\n # hostNamesLong = hostName.append(name + '.rf9.local')\n\nfor name in hostNames:\n hostNamesLong.append(name + '.rf9.local')\n print hostNamesLong\n \nprint \"@@{code.cmip}@@\"\nprint \"@@{code.cmPassword}@@\"\n\napi = ApiResource(\"@@{code.cmip}@@\", '7180', \"@@{code.cmAdmin}@@\", \"@@{code.cmPassword}@@\", version=\"@@{code.APIversion}@@\")\n\n#get the CM instance\ncm = api.get_cluster(\"@@{code.clusterName}@@\")\ncm = ClouderaManager(api)\n\n\ncmd = cm.host_install(\"@@{code.hostAdmin}@@\", hostNamesLong, password=\"@@{code.hostPassword}@@\", cm_repo_url=cm_repo_url)\nprint \"Installing hosts. This might take a while.\"\nwhile cmd.success == None:\n sleep(15)\n cmd = cmd.fetch()\n\nif cmd.success != True:\n print \"cm_host_install failed: \" + cmd.resultMessage\n# exit(0)\nprint \"cm_host_install succeeded\"\n\ncluster = api.get_cluster(\"@@{code.clusterName}@@\")\n\nhostsadded = cluster.add_hosts(hostNamesLong)\n\nparcels_list = []\n# get and list all available parcels\nprint \"Available parcels:\"\nfor p in cluster.get_all_parcels():\n print p.version\n if p.version.startswith(\"@@{code.cmPackage}@@\") and p.product == \"CDH\":\n parcels_list.append(p)\n\nif len(parcels_list) == 0:\n print \"No \"\n print cdh_version\n print \" parcel found!\"\n exit(0)\n\ncdh_parcel = parcels_list[0]\n\nfor p in parcels_list:\n if p.version > cdh_parcel.version:\n cdh_parcel = p\n\nprint cdh_parcel.product + ' ' + cdh_parcel.version + \" downloaded\"\n\n# distribute the parcel\nprint \"Starting parcel distribution. This might take a while.\"\ncmd = cdh_parcel.start_distribution()\nif cmd.success != True:\n print \"Parcel distribution failed!\"\n exit(0)\n\nprint 'wait 4 min'\n# make sure the distribution finishes\n#while cdh_parcel.stage != \"DISTRIBUTED\":\nsleep(200)\n# cdh_parcel = get_parcel(api, cdh_parcel.product, '5.13.3-1.cdh5.13.3.p0.2', cluster_name)\n\nprint cdh_parcel.product + ' ' + cdh_parcel.version + \" distributed\"\n\n# activate the parcel\ncmd = cdh_parcel.activate()\nif cmd.success != True:\n print \"Parcel activation failed!\"\n exit(0)\n\n# make sure the activation finishes\n#while cdh_parcel.stage != \"ACTIVATED\":\n# cdh_parcel = get_parcel(api, cdh_parcel.product, '5.13.3-1.cdh5.13.3.p0.2', cluster_name)\n\nprint cdh_parcel.product + ' ' + cdh_parcel.version + \" activated\"\n\n\nyarn = cluster.get_service('yarn')\nprint yarn.name\nhostsid = api.get_all_hosts()\n#range = len(hostsid)\n#print 'range: ' + range\n\nfor current in hostsid:\n if current.rackId == '/default/default':\n print 'host name to add' + current.hostname\n currentshort1 = current.hostname\n currentshort = currentshort1.split(\".\", 1)\n short = currentshort[0]\n print short\n yarn.create_role(short, 'NODEMANAGER', current.hostId)\n\ncluster.restart(restart_only_stale_services=True, redeploy_client_configuration=True)\n\nexit(0)", "script_type": "sh", "type": "", "command_line_args": "", "login_credential_local_reference": {"kind": "app_credential", "name": "go", "uuid": "f356d2b6-2fac-453c-84ad-d91bc21bb1ba"}}, "timeout_secs": "", "type": "EXEC", "variable_list": [], "uuid": "b24dd948-e65d-2c1e-52d8-c4d21d615274"}], "description": "", "name": "7da2879a_runbook_cloned_1", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "6405be68_dag", "uuid": "7ec268af-4e53-3c55-930a-740501d41cfb"}, "message_list": [], "variable_list": [], "uuid": "2363d280-af23-b97a-557f-1e6a3d156fbb"}, "type": "", "uninstall_runbook": {"task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera", "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}, "description": "", "message_list": [], "child_tasks_local_reference_list": [], "name": "de4520ca_dag", "state": "ACTIVE", "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "uuid": "aabae04b-08bb-466e-76a2-1fcfc3c816cd"}], "description": "", "name": "339b443d_runbook_cloned_1", "state": "ACTIVE", "main_task_local_reference": {"kind": "app_task", "name": "de4520ca_dag", "uuid": "aabae04b-08bb-466e-76a2-1fcfc3c816cd"}, "message_list": [], "variable_list": [], "uuid": "ff185ae5-bc26-095b-6840-9fc957638621"}}, "variable_list": [], "uuid": "44bbdab6-b71e-8989-d19b-aeb8c940122c"}]}, "name": "computeexpand"}, "spec": {"description": "", "resources": {"service_definition_list": [{"singleton": false, "name": "compute", "action_list": [{"critical": false, "type": "system", "description": "System action for creating an application", "name": "action_create", "runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_service", "name": "compute"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "b3c287b5_dag"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "b3c287b5_dag"}, "name": "eba78c76_runbook"}}, {"critical": false, "type": "system", "description": "System action for deleting an application. Deletes physical machines as well", "name": "action_delete", "runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_service", "name": "compute"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "1744848a_dag"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "1744848a_dag"}, "name": "740b4861_runbook"}}, {"critical": false, "type": "system", "description": "System action for starting an application", "name": "action_start", "runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_service", "name": "compute"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "87c10021_dag"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "87c10021_dag"}, "name": "3cedb317_runbook"}}, {"critical": false, "type": "system", "description": "System action for stopping an application", "name": "action_stop", "runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_service", "name": "compute"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "c68f8d5e_dag"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "c68f8d5e_dag"}, "name": "5be55dea_runbook"}}, {"critical": false, "type": "system", "description": "System action for restarting an application", "name": "action_restart", "runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_service", "name": "compute"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "5869d3a7_dag"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "5869d3a7_dag"}, "name": "4ebd0004_runbook"}}], "description": "", "port_list": [], "tier": "", "variable_list": [], "depends_on_list": []}, {"singleton": false, "name": "code", "action_list": [{"critical": false, "type": "system", "description": "System action for creating an application", "name": "action_create", "runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_service", "name": "code"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "b3c287b5_dag"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "b3c287b5_dag"}, "name": "eba78c76_runbook_cloned_1"}}, {"critical": false, "type": "system", "description": "System action for deleting an application. Deletes physical machines as well", "name": "action_delete", "runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_service", "name": "code"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "1744848a_dag"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "1744848a_dag"}, "name": "740b4861_runbook_cloned_1"}}, {"critical": false, "type": "system", "description": "System action for starting an application", "name": "action_start", "runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_service", "name": "code"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "87c10021_dag"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "87c10021_dag"}, "name": "3cedb317_runbook_cloned_1"}}, {"critical": false, "type": "system", "description": "System action for stopping an application", "name": "action_stop", "runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_service", "name": "code"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "c68f8d5e_dag"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "c68f8d5e_dag"}, "name": "5be55dea_runbook_cloned_1"}}, {"critical": false, "type": "system", "description": "System action for restarting an application", "name": "action_restart", "runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_service", "name": "code"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "5869d3a7_dag"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "5869d3a7_dag"}, "name": "4ebd0004_runbook_cloned_1"}}], "description": "", "port_list": [], "tier": "", "variable_list": [{"val_type": "STRING", "description": "", "value": "admin", "label": "", "attrs": {"type": ""}, "type": "LOCAL", "name": "cmAdmin"}, {"val_type": "STRING", "description": "", "value": "10.15.80.180", "label": "", "attrs": {"type": ""}, "type": "LOCAL", "name": "cmip"}, {"val_type": "STRING", "description": "", "value": "", "label": "", "attrs": {"is_secret_modified": false, "secret_reference": {}, "type": ""}, "type": "SECRET", "name": "cmPassword"}, {"val_type": "STRING", "description": "", "value": "18", "label": "", "attrs": {"type": ""}, "type": "LOCAL", "name": "APIversion"}, {"val_type": "STRING", "description": "", "value": "Cluster 1", "label": "", "attrs": {"type": ""}, "type": "LOCAL", "name": "clusterName"}, {"val_type": "STRING", "description": "", "value": "5.13.3-1.cdh5.13.3", "label": "", "attrs": {"type": ""}, "type": "LOCAL", "name": "cmPackage"}, {"val_type": "STRING", "description": "", "value": "", "label": "", "attrs": {"is_secret_modified": false, "secret_reference": {}, "type": ""}, "type": "SECRET", "name": "hostPassword"}, {"val_type": "STRING", "description": "", "value": "root", "label": "", "attrs": {"type": ""}, "type": "LOCAL", "name": "hostAdmin"}], "depends_on_list": []}], "substrate_definition_list": [{"description": "", "action_list": [], "readiness_probe": {"connection_type": "SSH", "address": "@@{platform.status.resources.nic_list[0].ip_endpoint_list[0].ip}@@", "disable_readiness_probe": false, "timeout_secs": "60", "connection_port": 22}, "editables": {"create_spec": {"resources": {}}}, "os_type": "Linux", "type": "AHV_VM", "create_spec": {"backup_policy": null, "type": "", "name": "yarn@@{calm_application_name}@@-@@{calm_random}@@", "resources": {"hardware_clock_timezone": "", "type": "", "boot_config": null, "power_state": "ON", "guest_tools": null}, "availability_zone_reference": null}, "variable_list": [], "name": "yarn"}, {"description": "", "action_list": [], "readiness_probe": {"connection_type": "SSH", "address": "@@{platform.status.resources.nic_list[0].ip_endpoint_list[0].ip}@@", "disable_readiness_probe": false, "timeout_secs": "60", "connection_port": 22}, "editables": {"create_spec": {"resources": {}}}, "os_type": "Linux", "type": "AHV_VM", "create_spec": {"backup_policy": null, "type": "", "name": "run@@{calm_application_name}@@-@@{calm_random}@@", "resources": {"hardware_clock_timezone": "", "type": "", "boot_config": {"boot_device": {"type": "", "disk_address": {"adapter_type": "SCSI", "device_index": 0, "type": ""}}, "type": "", "mac_address": ""}, "power_state": "ON", "guest_tools": null}, "availability_zone_reference": null}, "variable_list": [], "name": "runner"}], "credential_definition_list": [{"username": "root", "secret": {"attrs": {"is_secret_modified": false, "secret_reference": {}}}, "type": "PASSWORD", "description": "", "name": "go"}], "package_definition_list": [{"description": "", "action_list": [], "service_local_reference_list": [{"kind": "app_service", "name": "compute"}], "version": "", "type": "DEB", "options": {"install_runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "deployCPUa"}, "description": "", "child_tasks_local_reference_list": [{"kind": "app_task", "name": "PackageInstallTask"}], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "6405be68_dag"}, {"target_any_local_reference": {"kind": "app_package", "name": "deployCPUa"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"exit_status": [], "script": "#!/usr/bin/env bash\n\nhostnamectl set-hostname @@{name}@@.rf9.local\n\nsystemctl restart network.service", "script_type": "sh", "type": "", "command_line_args": "", "login_credential_local_reference": {"kind": "app_credential", "name": "go"}}, "timeout_secs": "", "type": "EXEC", "variable_list": [], "name": "PackageInstallTask"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "6405be68_dag"}, "name": "7da2879a_runbook"}, "type": "", "uninstall_runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "deployCPUa"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "de4520ca_dag"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "de4520ca_dag"}, "name": "339b443d_runbook"}}, "variable_list": [], "name": "deployCPUa"}, {"description": "", "action_list": [], "service_local_reference_list": [{"kind": "app_service", "name": "code"}], "version": "", "type": "DEB", "options": {"install_runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera"}, "description": "", "child_tasks_local_reference_list": [{"kind": "app_task", "name": "PackageInstallTask"}], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "6405be68_dag"}, {"target_any_local_reference": {"kind": "app_package", "name": "cloudera"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"exit_status": [], "script": "#!/usr/bin/env python\nfrom cm_api.api_client import ApiResource\nfrom cm_api.endpoints.clusters import ApiCluster\nfrom cm_api.endpoints.clusters import create_cluster\nfrom cm_api.endpoints.parcels import ApiParcel\nfrom cm_api.endpoints.parcels import get_parcel\nfrom cm_api.endpoints.cms import ClouderaManager\nfrom cm_api.endpoints.services import ApiService, ApiServiceSetupInfo\nfrom cm_api.endpoints.services import create_service\nfrom cm_api.endpoints.types import ApiCommand, ApiRoleConfigGroupRef\nfrom cm_api.endpoints.role_config_groups import get_role_config_group\nfrom cm_api.endpoints.role_config_groups import ApiRoleConfigGroup\nfrom cm_api.endpoints.roles import ApiRole\nfrom time import sleep\n\nhost_username = 'root'\ncm_repo_url = None\ncdh_version = \"CDH5\" # also valid: \"CDH4\"\ncdh_version_number = \"5\"\n#240\nsleep(240)\n\nhostsIP = [ ]\nhostNames = [ ]\nhostNamesLong = [ ]\nthostNames = [ ]\nthostNames = \"@@{compute.name}@@\"\n#hostsIP = thostsIP.split(,)\nhostNames = thostNames.split(\",\")\n\n#for name in hostName:\n # hostNamesLong = hostName.append(name + '.rf9.local')\n\nfor name in hostNames:\n hostNamesLong.append(name + '.rf9.local')\n print hostNamesLong\n \nprint \"@@{code.cmip}@@\"\nprint \"@@{code.cmPassword}@@\"\n\napi = ApiResource(\"@@{code.cmip}@@\", '7180', \"@@{code.cmAdmin}@@\", \"@@{code.cmPassword}@@\", version=\"@@{code.APIversion}@@\")\n\n#get the CM instance\ncm = api.get_cluster(\"@@{code.clusterName}@@\")\ncm = ClouderaManager(api)\n\n\ncmd = cm.host_install(\"@@{code.hostAdmin}@@\", hostNamesLong, password=\"@@{code.hostPassword}@@\", cm_repo_url=cm_repo_url)\nprint \"Installing hosts. This might take a while.\"\nwhile cmd.success == None:\n sleep(15)\n cmd = cmd.fetch()\n\nif cmd.success != True:\n print \"cm_host_install failed: \" + cmd.resultMessage\n# exit(0)\nprint \"cm_host_install succeeded\"\n\ncluster = api.get_cluster(\"@@{code.clusterName}@@\")\n\nhostsadded = cluster.add_hosts(hostNamesLong)\n\nparcels_list = []\n# get and list all available parcels\nprint \"Available parcels:\"\nfor p in cluster.get_all_parcels():\n print p.version\n if p.version.startswith(\"@@{code.cmPackage}@@\") and p.product == \"CDH\":\n parcels_list.append(p)\n\nif len(parcels_list) == 0:\n print \"No \"\n print cdh_version\n print \" parcel found!\"\n exit(0)\n\ncdh_parcel = parcels_list[0]\n\nfor p in parcels_list:\n if p.version > cdh_parcel.version:\n cdh_parcel = p\n\nprint cdh_parcel.product + ' ' + cdh_parcel.version + \" downloaded\"\n\n# distribute the parcel\nprint \"Starting parcel distribution. This might take a while.\"\ncmd = cdh_parcel.start_distribution()\nif cmd.success != True:\n print \"Parcel distribution failed!\"\n exit(0)\n\nprint 'wait 4 min'\n# make sure the distribution finishes\n#while cdh_parcel.stage != \"DISTRIBUTED\":\nsleep(200)\n# cdh_parcel = get_parcel(api, cdh_parcel.product, '5.13.3-1.cdh5.13.3.p0.2', cluster_name)\n\nprint cdh_parcel.product + ' ' + cdh_parcel.version + \" distributed\"\n\n# activate the parcel\ncmd = cdh_parcel.activate()\nif cmd.success != True:\n print \"Parcel activation failed!\"\n exit(0)\n\n# make sure the activation finishes\n#while cdh_parcel.stage != \"ACTIVATED\":\n# cdh_parcel = get_parcel(api, cdh_parcel.product, '5.13.3-1.cdh5.13.3.p0.2', cluster_name)\n\nprint cdh_parcel.product + ' ' + cdh_parcel.version + \" activated\"\n\n\nyarn = cluster.get_service('yarn')\nprint yarn.name\nhostsid = api.get_all_hosts()\n#range = len(hostsid)\n#print 'range: ' + range\n\nfor current in hostsid:\n if current.rackId == '/default/default':\n print 'host name to add' + current.hostname\n currentshort1 = current.hostname\n currentshort = currentshort1.split(\".\", 1)\n short = currentshort[0]\n print short\n yarn.create_role(short, 'NODEMANAGER', current.hostId)\n\ncluster.restart(restart_only_stale_services=True, redeploy_client_configuration=True)\n\nexit(0)", "script_type": "sh", "type": "", "command_line_args": "", "login_credential_local_reference": {"kind": "app_credential", "name": "go"}}, "timeout_secs": "", "type": "EXEC", "variable_list": [], "name": "PackageInstallTask"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "6405be68_dag"}, "name": "7da2879a_runbook_cloned_1"}, "type": "", "uninstall_runbook": {"variable_list": [], "task_definition_list": [{"target_any_local_reference": {"kind": "app_package", "name": "cloudera"}, "description": "", "child_tasks_local_reference_list": [], "attrs": {"edges": [], "type": ""}, "timeout_secs": "", "type": "DAG", "variable_list": [], "name": "de4520ca_dag"}], "description": "", "main_task_local_reference": {"kind": "app_task", "name": "de4520ca_dag"}, "name": "339b443d_runbook_cloned_1"}}, "variable_list": [], "name": "cloudera"}], "app_profile_list": [{"deployment_create_list": [{"name": "76df5aad_deployment", "action_list": [], "package_local_reference_list": [{"kind": "app_package", "name": "deployCPUa"}], "editables": {"min_replicas": true}, "max_replicas": "1", "substrate_local_reference": {"kind": "app_substrate", "name": "yarn"}, "min_replicas": "1", "variable_list": [], "description": ""}, {"description": "", "action_list": [], "package_local_reference_list": [{"kind": "app_package", "name": "cloudera"}], "max_replicas": "1", "substrate_local_reference": {"kind": "app_substrate", "name": "runner"}, "min_replicas": "1", "variable_list": [], "name": "76df5aad_deployment_cloned_1"}], "variable_list": [], "description": "", "action_list": [], "name": "Default1"}], "default_credential_local_reference": {"kind": "app_credential", "name": "go"}, "client_attrs": {"None": {"y": 450, "x": 810}, "code": {"y": 380, "x": 760}, "compute": {"y": 380, "x": 580}}}, "name": "computeexpand"}, "api_version": "3.0", "metadata": {"last_update_time": "1526503361119604", "creation_time": "1520027297574039", "kind": "blueprint", "spec_version": 85, "name": "computeexpand"}}

Speak Your Mind

*