o
    .&ßa'e  ã                   @   sÜ   d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlm	Z	 d d	lm
Z
 d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ G dd„ deƒZdS )é    N)ÚBasicCommand)Úapplicationutils)Úargumentschema)Ú	constants)Ú
emrfsutils)Úemrutils)Ú
exceptions)Ú
hbaseutils)Úhelptext)Úinstancegroupsutils)Úinstancefleetsutils)Ú	steputils)ÚCommand)ÚEC2_ROLE_NAME)ÚEMR_ROLE_NAME)Újsonc                   @   s²  e Zd ZdZejZg dejdœ‘dejdœ‘de	j
ejdœ‘dejdœ‘dejdœ‘d	d
dejdœ‘dd
ddœ‘de	jejdœ‘ddejdœ‘dejdœ‘dejdœ‘dejdœ‘dejdœ‘dd
ejdœ‘dejdœ‘deje	jdœ‘dd
dejdœ‘dd
ddœ‘dejdœ‘d d
d!ejdœ‘d"d
d!dœ‘d#d
d$ejdœ‘d%d
d$dœ‘d&d'eje	jd(œ‘d)ej e	j!dœ‘d*ej"e	j#dœ‘d+ej$e	j%dœ‘d,e	j&ej'dœ‘d-ej(dœ‘d.e	j)ej*dœ‘d/ej+dœ‘d0ej,dœ‘d1ej-dœ‘d2ej.dœ‘d3e	j/ej0dœ‘d4d5ej1d6œ‘d7e	j2ej3dœ‘d8e	j4ej5dœ‘d9e	j6ej7dœ‘Z8e9 :d:d;¡Z;e9 :d:d<¡Z<d=d>„ Z=d?d@„ Z>dAdB„ Z?dCdD„ Z@dEdF„ ZAdGdH„ ZBdIdJ„ ZCdKdL„ ZDdMdN„ ZEdOdP„ ZFdQdR„ ZGdSdT„ ZHdUS )VÚCreateClusterzcreate-clusterzrelease-label)ÚnameÚ	help_textzami-versionzinstance-groups)r   Úschemar   zinstance-typezinstance-countzauto-terminateÚ
store_trueÚauto_terminate)r   ÚactionÚ
group_namer   zno-auto-terminate)r   r   r   zinstance-fleetsr   zDevelopment Cluster)r   Údefaultr   zlog-urizlog-encryption-kms-key-idzservice-rolezauto-scaling-rolezuse-default-roles)r   r   r   Úconfigurationszec2-attributes)r   r   r   ztermination-protectedÚtermination_protectedzno-termination-protectedzscale-down-behaviorzvisible-to-all-usersÚ
visibilityzno-visible-to-all-userszenable-debuggingÚdebugzno-enable-debuggingÚtagsú+)r   Únargsr   r   zbootstrap-actionsÚapplicationsÚemrfsÚstepszadditional-infozrestore-from-hbase-backupzsecurity-configurationzcustom-ami-idzebs-root-volume-sizezrepo-upgrade-on-bootzkerberos-attributeszstep-concurrency-levelÚinteger)r   Úcli_type_namer   zmanaged-scaling-policyzplacement-group-configszauto-termination-policyÚemrzcreate-cluster-synopsis.txtzcreate-cluster-examples.rstc                 C   sÄ  i }|j |d< |  |¡ d}|jdu r |jd ur tjdd|d�‚|jdu r7|jd ur7d|jv r7tjdd|d�‚|jd urH|jd urHtjd	d
d�‚i }|jd urXt	 
|j¡|d< ntj|j|j|jd�|d< |jd urŠ|j|d< |jd urŠz
t |j¡|d< W n ty‰   tdƒ‚w |jd u r«|jd ur«t d|j¡}|d u r¦tj|jd�‚|j|d< t |d|j¡ t |d|j¡ |jd urÈt |d|j¡ |jdu rÝt|_|jd u rØi |_t|jd< t |d|j¡ |jd u�r|d D ]}d| ¡ v �r|j d u �rt !¡ ‚qït |d|j ¡ |j"d u�rt |d|j"¡ |j#du �r(|j$du �r(d|_#t %|j#d|j$d¡|d< t %|j&d |j'd!¡|d"< |j(du �rO|j)du �rOd|_(t %|j(d#|j)d$¡|d%< t *|j+¡|d&< ||d'< |jd u�ru| j,||jd(� t %|j-d)|j.d*¡}|jd u �r�|du �r�tj/‚|du �rŸ| j0|d+|  1||¡gd,� |j2d u�râ|jd u �rÐt3j4| j5|j2|d d-�\}	}
}|  0|d.|	¡ |  0|d/|
¡ |  0|d+|¡ ng |d0< |j2D ]
}|d0  6|¡ �q×|j7}|d u�rt8j9| :d1¡| :d2¡d3�}tj;t<j=t<j>t<j?|d4�}|  0|d+|g¡ |j@d u�r| jA||j@d5� |jBd u�r)| jC||jB|jd6� |jDd u�rBtEjF|jD| j5|jd7�}| j0|d+|d,� |jGd u�rPt |d8|jG¡ |jHd u�r^t |d9|jH¡ |jId u�rnt |d:tJ|jIƒ¡ |jKd u�r|t |d;|jK¡ |jLd u�rŠt |d<|jL¡ |jMd u�r•|jM|d=< |jNd u�r£t |d>|jN¡ |jOd u�r±t |d?|jO¡ |jPd u�r¿t |d@|jP¡ |  Q|¡ t R| jSdA|| j5|jT|jU¡}|  V|¡}t W| jSdA||¡ dBS )CNÚNamez| Either choose --use-default-roles or use both --service-role <roleName> and --ec2-attributes InstanceProfile=<profileName>.Tz--use-default-rolesz--service-role)Úoption1Úoption2ÚmessageÚInstanceProfilez --ec2-attributes InstanceProfilez--instance-groupsz--instance-fleets©r)   r*   ÚInstanceFleets)Úinstance_groupsÚinstance_typeÚinstance_countÚInstanceGroupsÚReleaseLabelÚConfigurationsz=aws: error: invalid json argument for option --configurationsz\d?\..*)Úami_versionÚ
AmiVersionÚAdditionalInfoÚLogUriÚLogEncryptionKmsKeyIdÚServiceRoleÚAutoScalingPolicyÚAutoScalingRoleÚScaleDownBehaviorFz--no-auto-terminatez--auto-terminateÚKeepJobFlowAliveWhenNoStepsz--termination-protectedz--no-termination-protectedÚTerminationProtectedz--visible-to-all-usersz--no-visible-to-all-usersÚVisibleToAllUsersÚTagsÚ	Instances)ÚclusterÚparsed_attrsz--enable-debuggingz--no-enable-debuggingÚSteps©rC   ÚkeyÚvalue)ÚregionÚparsed_applicationsr5   ÚNewSupportedProductsÚBootstrapActionsÚApplicationsÚDirÚBackupVersion)ÚdirÚbackup_version)Újarr   Úaction_on_failureÚargs)rC   Úparsed_boostrap_actions)rC   Ú
emrfs_argsÚrelease_label)Úparsed_step_listrI   rW   ÚSecurityConfigurationÚCustomAmiIdÚEbsRootVolumeSizeÚRepoUpgradeOnBootÚKerberosAttributesÚStepConcurrencyLevelÚManagedScalingPolicyÚPlacementGroupConfigsÚAutoTerminationPolicyÚrun_job_flowr   )Xr   Ú#_validate_release_label_ami_versionÚuse_default_rolesÚservice_roler   ÚMutualExclusiveOptionErrorÚec2_attributesr/   Úinstance_fleetsr   Ú"validate_and_build_instance_fleetsr   Ú"validate_and_build_instance_groupsr0   r1   rW   r   r   ÚloadsÚ
ValueErrorr5   ÚreÚmatchÚInvalidAmiVersionErrorr   Ú
apply_dictÚadditional_infoÚlog_uriÚlog_encryption_kms_key_idr   r   ÚkeysÚauto_scaling_roleÚMissingAutoScalingRoleErrorÚscale_down_behaviorÚno_auto_terminater   Úapply_boolean_optionsr   Úno_termination_protectedÚvisible_to_all_usersÚno_visible_to_all_usersÚ
parse_tagsr   Ú_build_ec2_attributesÚenable_debuggingÚno_enable_debuggingÚLogUriErrorÚ_update_cluster_dictÚ_build_enable_debuggingr"   r   Úbuild_applicationsrI   ÚappendÚrestore_from_hbase_backupr	   Ú$build_hbase_restore_from_backup_argsÚgetÚ
build_stepr   ÚHBASE_JAR_PATHÚHBASE_RESTORE_STEP_NAMEÚCANCEL_AND_WAITÚbootstrap_actionsÚ_build_bootstrap_actionsr#   Ú_handle_emrfs_parametersr$   r   Úbuild_step_config_listÚsecurity_configurationÚcustom_ami_idÚebs_root_volume_sizeÚintÚrepo_upgrade_on_bootÚkerberos_attributesÚstep_concurrency_levelÚmanaged_scaling_policyÚplacement_group_configsÚauto_termination_policyÚ_validate_required_applicationsÚcallÚ_sessionÚendpoint_urlÚ
verify_sslÚ_construct_resultÚdisplay_response)ÚselfÚparsed_argsÚparsed_globalsÚparamsÚservice_role_validation_messageÚinstances_configÚis_valid_ami_versionÚinstance_groupÚdebugging_enabledÚapp_listÚba_listÚ	step_listÚapplicationÚhbase_restore_configrT   Ústep_configÚ
steps_listÚrun_job_flow_responseÚconstructed_result© r´   úI/usr/lib/python3/dist-packages/awscli/customizations/emr/createcluster.pyÚ_run_main_commandŠ   sÈ  

ÿ

ý


ý

þ
ÿÿýÿ


ÿÿ

ÿ
ÿ
ÿ


€üÿüÿüÿÿü

ÿýýÿÿÿÿ

þüÿþýýÿÿÿÿÿÿ
ÿþþ
þ

ÿzCreateCluster._run_main_commandc                 C   s:   d }d }|d ur|  d¡}|  d¡}|d ur||dœS i S )NÚ	JobFlowIdÚ
ClusterArn)Ú	ClusterIdr¸   )rˆ   )r¢   Úrun_job_flow_resultÚ	jobFlowIdÚ
clusterArnr´   r´   rµ   r    {  s   

ÿzCreateCluster._construct_resultc                 C   sx  |  ¡ }|d }d|v rd|v rtjddd�‚d|v r&d|v r&tjddd�‚d|v s.d|v r9d|v s6d|v r9tj‚tj|d|dd	� tj|d|d
d	� tj|d|dd	� d|v rhtƒ |d< tj|d|d dd	� d|v r|tƒ |d< tj|d|d dd	� tj|d|dd	� tj|d|dd	� tj|d|dd	� tj|d|dd	� tj|d|dd	� tj|d|dd	� tj|d|d� |S )NrB   ÚSubnetIdÚ	SubnetIdsr-   ÚAvailabilityZoneÚAvailabilityZonesÚKeyNameÚ
Ec2KeyName©Ú
src_paramsÚsrc_keyÚdest_paramsÚdest_keyÚEc2SubnetIdÚEc2SubnetIdsÚ	Placementr,   ÚJobFlowRoleÚEmrManagedMasterSecurityGroupÚEmrManagedSlaveSecurityGroupÚServiceAccessSecurityGroupÚAdditionalMasterSecurityGroupsÚAdditionalSlaveSecurityGroups©r¥   rG   rH   )rt   r   rf   ÚSubnetAndAzValidationErrorr   Úapply_paramsÚdictÚapply)r¢   rC   rD   rt   Ú	instancesr´   r´   rµ   r~   ˆ  s†   þþþþþ
ý
ýþþþþþþz#CreateCluster._build_ec2_attributesc           	      C   sÐ   |  d¡}|d u rg }g }t|ƒt|ƒ tjkrtdƒ‚|D ]9}i }|  d¡d ur1|  d¡|d< ntj|d< i }tj|d|dd� tj|d|dd� tj|d|d� | 	|¡ q|| }t|ƒd	krf||d< |S )
NrL   zGaws: error: maximum number of bootstrap actions for a cluster exceeded.r(   ÚPathrÃ   ÚArgsÚScriptBootstrapActionrÑ   r   )
rˆ   Úlenr   ÚMAX_BOOTSTRAP_ACTION_NUMBERrl   ÚBOOTSTRAP_ACTION_NAMEr   rÓ   rÕ   r…   )	r¢   rC   rU   Úcluster_ba_listr�   ÚbaÚ	ba_configÚscript_arg_configÚresultr´   r´   rµ   rŽ   Î  s@   
ÿ
þþýz&CreateCluster._build_bootstrap_actionsc                 C   sL   |j rtj}tjg}nt | j¡}tjtj| jd�g}tj	tj
tj||d�S )N)Úrelative_pathrI   )r   rS   rR   rT   )rW   r   ÚCOMMAND_RUNNERÚDEBUGGING_COMMANDr   Úget_script_runnerrI   Úbuild_s3_linkÚDEBUGGING_PATHr‰   ÚDEBUGGING_NAMEÚTERMINATE_CLUSTER)r¢   r£   r¤   rR   rT   r´   r´   rµ   rƒ   ó  s   
þüz%CreateCluster._build_enable_debuggingc                 C   s@   ||  ¡ v r||  |7  < |S |d urt|ƒdkr|||< |S )Nr   )rt   rÚ   )r¢   rC   rG   rH   r´   r´   rµ   r‚     s   þz"CreateCluster._update_cluster_dictc                 C   sL   |j d ur|jd urtjddd�‚|j d u r"|jd u r$tjddd�‚d S d S )Nz--ami-versionz--release-labelr-   )r5   rW   r   rf   ÚRequiredOptionsError)r¢   r£   r´   r´   rµ   rc   
  s   

þ

þþz1CreateCluster._validate_release_label_ami_versionc                 C   sr   t g ƒ}|jd urt dd„ |jD ƒƒ}|  ||¡}|jd ur+tj|vr+| tj ¡ ¡ t|ƒdkr7t	j
|d�‚d S )Nc                 S   s   g | ]}|d    ¡ ‘qS )r(   )Úlower)Ú.0Úappr´   r´   rµ   Ú
<listcomp>  s    zACreateCluster._validate_required_applications.<locals>.<listcomp>r   )r"   )Úsetr"   Ú#_get_missing_applications_for_stepsr†   r   ÚHBASEÚaddÚtitlerÚ   r   ÚMissingApplicationsError)r¢   r£   Úspecified_appsÚmissing_appsr´   r´   rµ   r›     s   
ÿÿ

ÿÿz-CreateCluster._validate_required_applicationsc                 C   sˆ   t tjtjtjgƒ}t g ƒ}|jd urB|jD ]+}t|ƒt|ƒkr# |S | d¡}|d urA| ¡ }||v rA||vrA| 	|d  
¡ ¡ q|S )NÚType)rï   r   ÚHIVEÚPIGÚIMPALAr$   rÚ   rˆ   rë   rò   ró   )r¢   rõ   r£   Úallowed_app_stepsrö   ÚstepÚ	step_typer´   r´   rµ   rð   +  s"   
ÿ


ù€z1CreateCluster._get_missing_applications_for_stepsc                 C   s   |j r
dd„ |D ƒ}|S )Nc                 S   s$   g | ]}|j d kr|j dkr|‘qS )re   Úinstance_profile)r   )rì   Úxr´   r´   rµ   rî   ?  s
    

þzICreateCluster._filter_configurations_in_special_cases.<locals>.<listcomp>)rd   )r¢   r   r£   Úparsed_configsr´   r´   rµ   Ú'_filter_configurations_in_special_cases<  s   z5CreateCluster._filter_configurations_in_special_casesc                 C   sP   |r|   |¡ t |¡}| j|d|gd� d S t | j|¡}| j|d|d� d S )Nr4   rF   rL   )Úvalidate_no_emrfs_configurationr   Úbuild_emrfs_confiurationr‚   Úbuild_bootstrap_action_configsrI   )r¢   rC   rV   rW   Úemrfs_configurationÚemrfs_ba_config_listr´   r´   rµ   r�   D  s    
ÿ
þÿ
þz&CreateCluster._handle_emrfs_parametersc                 C   s<   d|v r|d D ]}|d ur|  d¡tjkrtj‚qd S d S )Nr4   ÚClassification)rˆ   r   Ú
EMRFS_SITEr   Ú DuplicateEmrFsConfigurationError)r¢   rC   Úconfigr´   r´   rµ   r  T  s   €üz-CreateCluster.validate_no_emrfs_configurationN)IÚ__name__Ú
__module__Ú__qualname__ÚNAMEr
   ÚCREATE_CLUSTER_DESCRIPTIONÚDESCRIPTIONÚRELEASE_LABELÚAMI_VERSIONr   ÚINSTANCE_GROUPS_SCHEMAÚINSTANCE_GROUPSÚINSTANCE_TYPEÚINSTANCE_COUNTÚAUTO_TERMINATEÚINSTANCE_FLEETS_SCHEMAÚINSTANCE_FLEETSÚCLUSTER_NAMEÚLOG_URIÚLOG_ENCRYPTION_KMS_KEY_IDÚSERVICE_ROLEÚAUTOSCALING_ROLEÚUSE_DEFAULT_ROLESÚCONFIGURATIONSÚEC2_ATTRIBUTESÚEC2_ATTRIBUTES_SCHEMAÚTERMINATION_PROTECTEDÚSCALE_DOWN_BEHAVIORÚ
VISIBILITYÚ	DEBUGGINGÚTAGSÚTAGS_SCHEMAÚBOOTSTRAP_ACTIONSÚBOOTSTRAP_ACTIONS_SCHEMAÚAPPLICATIONSÚAPPLICATIONS_SCHEMAÚEMR_FSÚEMR_FS_SCHEMAÚSTEPS_SCHEMAÚSTEPSÚADDITIONAL_INFOÚ HBASE_RESTORE_FROM_BACKUP_SCHEMAÚRESTORE_FROM_HBASEÚSECURITY_CONFIGÚCUSTOM_AMI_IDÚEBS_ROOT_VOLUME_SIZEÚREPO_UPGRADE_ON_BOOTÚKERBEROS_ATTRIBUTES_SCHEMAÚKERBEROS_ATTRIBUTESÚSTEP_CONCURRENCY_LEVELÚMANAGED_SCALING_POLICY_SCHEMAÚMANAGED_SCALING_POLICYÚPLACEMENT_GROUP_CONFIGS_SCHEMAÚPLACEMENT_GROUP_CONFIGSÚAUTO_TERMINATION_POLICY_SCHEMAÚAUTO_TERMINATION_POLICYÚ	ARG_TABLEr   Ú	FROM_FILEÚSYNOPSISÚEXAMPLESr¶   r    r~   rŽ   rƒ   r‚   rc   r›   rð   r  r�   r  r´   r´   r´   rµ   r   !   s„   ÿÿÿýþûÿø
ÿöþôÿñþïþìÿéÿçÿåÿãÿá!ÿß#þÝ&þÚ)ÿ×+ÿÕ-þÓ0ÿÐ2þÎ5ÿË7þÉ:þÆ=þÃ@þÀCþ½FÿºHþ¸KÿµMÿ³Oÿ±Qÿ¯Sþ­VþªYþ§\þ¤_þ¡c rF%r   )rm   Úawscli.customizations.commandsr   Úawscli.customizations.emrr   r   r   r   r   r   r	   r
   r   r   r   Ú!awscli.customizations.emr.commandr   Ú#awscli.customizations.emr.constantsr   r   Úbotocore.compatr   r   r´   r´   r´   rµ   Ú<module>   s$   