o
    .&ßa[  ã                   @   s:  d dl Z d dlZd dlmZ d dlmZmZmZmZm	Z	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlm Z  d dlm!Z! d dlm"Z" d dlm#Z# d dlm$Z$ d dlm%Z% d dlm&Z& d dlm'Z' d dl(m)Z) e  *e+¡Z,G d d!„ d!e-ƒZ.G d"d#„ d#e-ƒZ/G d$d%„ d%e-ƒZ0G d&d'„ d'e0ƒZ1G d(d)„ d)e0ƒZ2G d*d+„ d+e0ƒZ3G d,d-„ d-e1ƒZ4G d.d/„ d/e2ƒZ5G d0d1„ d1e0ƒZ6G d2d3„ d3e0ƒZ7dS )4é    N)ÚTransferManager)Úhuman_readable_sizeÚMAX_UPLOAD_SIZEÚfind_bucket_keyÚrelative_pathÚcreate_warningÚNonSeekableStream)Ú*create_transfer_config_from_runtime_config)ÚUploadResultSubscriber)ÚDownloadResultSubscriber)ÚCopyResultSubscriber)ÚUploadStreamResultSubscriber)ÚDownloadStreamResultSubscriber)ÚDeleteResultSubscriber)ÚQueuedResult)ÚSuccessResult)ÚFailureResult)ÚDryRunResult)ÚResultRecorder)ÚResultPrinter)ÚOnlyShowErrorsResultPrinter)ÚNoProgressResultPrinter)ÚResultProcessor)ÚCommandResultRecorder)ÚRequestParamsMapper©ÚStdoutBytesWriter)ÚProvideSizeSubscriber)Ú"ProvideUploadContentTypeSubscriber)Ú ProvideCopyContentTypeSubscriber)Ú!ProvideLastModifiedTimeSubscriber)ÚDirectoryCreatorSubscriber)ÚDeleteSourceFileSubscriber)ÚDeleteSourceObjectSubscriber)Ú DeleteCopySourceObjectSubscriber)Úget_binary_stdinc                   @   s(   e Zd ZdZdd„ Zdd„ Zdd„ ZdS )	ÚS3TransferHandlerFactoryé   c                 C   s   || _ || _dS )a  Factory for S3TransferHandlers

        :type cli_params: dict
        :param cli_params: The parameters provide to the CLI command

        :type runtime_config: RuntimeConfig
        :param runtime_config: The runtime config for the CLI command
            being run
        N)Ú_cli_paramsÚ_runtime_config)ÚselfÚ
cli_paramsÚruntime_config© r-   úD/usr/lib/python3/dist-packages/awscli/customizations/s3/s3handler.pyÚ__init__:   s   

z!S3TransferHandlerFactory.__init__c           	      C   sr   t | jƒ}| j|_| j|_t||ƒ}t d|j|j	¡ t
ƒ }|g}|  ||¡ t||ƒ}t|||ƒ}t|| j|ƒS )ag  Creates a S3TransferHandler instance

        :type client: botocore.client.Client
        :param client: The client to power the S3TransferHandler

        :type result_queue: queue.Queue
        :param result_queue: The result queue to be used to process results
            for the S3TransferHandler

        :returns: A S3TransferHandler instance
        z7Using a multipart threshold of %s and a part size of %s)r	   r)   ÚMAX_IN_MEMORY_CHUNKSÚmax_in_memory_upload_chunksÚmax_in_memory_download_chunksr   ÚLOGGERÚdebugÚmultipart_thresholdÚmultipart_chunksizer   Ú_add_result_printerr   r   ÚS3TransferHandlerr(   )	r*   ÚclientÚresult_queueÚtransfer_configÚtransfer_managerÚresult_recorderÚresult_processor_handlersÚresult_processorÚcommand_result_recorderr-   r-   r.   Ú__call__G   s0   ÿÿ
ýÿÿÿz!S3TransferHandlerFactory.__call__c                 C   sh   | j  d¡rd S | j  d¡rt|ƒ}n| j  d¡rt|ƒ}n| j  d¡s)t|ƒ}nt|ƒ}| |¡ d S )NÚquietÚonly_show_errorsÚ	is_streamÚprogress)r(   Úgetr   r   r   Úappend)r*   r=   r>   Úresult_printerr-   r-   r.   r7   k   s   


z,S3TransferHandlerFactory._add_result_printerN)Ú__name__Ú
__module__Ú__qualname__r0   r/   rA   r7   r-   r-   r-   r.   r&   7   s
    $r&   c                   @   s   e Zd Zdd„ Zdd„ ZdS )r8   c                 C   sP   || _ || _| j | jj|f}t|Ž t|Ž t|Ž t|Ž t|Ž t|Ž t	|Ž g| _
dS )a  Backend for performing S3 transfers

        :type transfer_manager: s3transfer.manager.TransferManager
        :param transfer_manager: Transfer manager to use for transfers

        :type cli_params: dict
        :param cli_params: The parameters passed to the CLI command in the
            form of a dictionary

        :type result_command_recorder: ResultCommandRecorder
        :param result_command_recorder: The result command recorder to be
            used to get the final result of the transfer
        N)Ú_transfer_managerÚ_result_command_recorderr:   ÚUploadStreamRequestSubmitterÚDownloadStreamRequestSubmitterÚUploadRequestSubmitterÚDownloadRequestSubmitterÚCopyRequestSubmitterÚDeleteRequestSubmitterÚLocalDeleteRequestSubmitterÚ_submitters)r*   r<   r+   Úresult_command_recorderÚsubmitter_argsr-   r-   r.   r/   z   s   
þ
ùzS3TransferHandler.__init__c              	   C   sœ   | j �> | j�+ d}|D ]}| jD ]}| |¡r#| |¡r!|d7 } nqq| j  |¡ W d  ƒ n1 s5w   Y  W d  ƒ n1 sDw   Y  | j  ¡ S )a…  Process iterable of FileInfos for transfer

        :type fileinfos: iterable of FileInfos
        param fileinfos: Set of FileInfos to submit to underlying transfer
            request submitters to make transfer API calls to S3

        :rtype: CommandResult
        :returns: The result of the command that specifies the number of
            failures and warnings encountered.
        r   é   N)rM   rL   rU   Ú
can_submitÚsubmitÚnotify_total_submissionsÚget_command_result)r*   Ú	fileinfosÚtotal_submissionsÚfileinfoÚ	submitterr-   r-   r.   ÚcallŸ   s$   


ý€ÿø€ÿ
zS3TransferHandler.callN)rI   rJ   rK   r/   ra   r-   r-   r-   r.   r8   y   s    %r8   c                   @   sŒ   e Zd ZdZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
d„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ ZdS ) ÚBaseTransferRequestSubmitterNc                 C   s   || _ || _|| _dS )a\  Submits transfer requests to the TransferManager

        Given a FileInfo object and provided CLI parameters, it will add the
        necessary extra arguments and subscribers in making a call to the
        TransferManager.

        :type transfer_manager: s3transfer.manager.TransferManager
        :param transfer_manager: The underlying transfer manager

        :type result_queue: queue.Queue
        :param result_queue: The result queue to use

        :type cli_params: dict
        :param cli_params: The associated CLI parameters passed in to the
            command as a dictionary.
        N)rL   Ú_result_queuer(   )r*   r<   r:   r+   r-   r-   r.   r/   ¼   s   
z%BaseTransferRequestSubmitter.__init__c                 C   s   |   |¡}|s|  |¡S dS )a¦  Submits a transfer request based on the FileInfo provided

        There is no guarantee that the transfer request will be made on
        behalf of the fileinfo as a fileinfo may be skipped based on
        circumstances in which the transfer is not possible.

        :type fileinfo: awscli.customizations.s3.fileinfo.FileInfo
        :param fileinfo: The FileInfo to be used to submit a transfer
            request to the underlying transfer manager.

        :rtype: s3transfer.futures.TransferFuture
        :returns: A TransferFuture representing the transfer if it the
            transfer was submitted. If it was not submitted nothing
            is returned.
        N)Ú_warn_and_signal_if_skipÚ
_do_submit)r*   r_   Úshould_skipr-   r-   r.   rZ   Ñ   s   

ÿz#BaseTransferRequestSubmitter.submitc                 C   ó   t dƒ‚)a�  Checks whether it can submit a particular FileInfo

        :type fileinfo: awscli.customizations.s3.fileinfo.FileInfo
        :param fileinfo: The FileInfo to check if the transfer request
            submitter can handle.

        :returns: True if it can use the provided FileInfo to make a transfer
            request to the underlying transfer manager. False, otherwise.
        zcan_submit()©ÚNotImplementedError©r*   r_   r-   r-   r.   rY   å   s   
z'BaseTransferRequestSubmitter.can_submitc                 C   sŒ   i }| j r|   || j¡ g }|  ||¡ | jr2d| ji}| j dd¡r'd|d< | | jdi |¤Ž¡ | j d¡s?|  |||¡S |  |¡ d S )Nr:   Úis_moveFÚmoveÚtransfer_typeÚdryrunr-   )	ÚREQUEST_MAPPER_METHODr(   Ú_add_additional_subscribersÚRESULT_SUBSCRIBER_CLASSrc   rF   rG   Ú_submit_transfer_requestÚ_submit_dryrun)r*   r_   Ú
extra_argsÚsubscribersÚresult_kwargsr-   r-   r.   re   ñ   s   
ÿz'BaseTransferRequestSubmitter._do_submitc                 C   s@   |j }| j dd¡rd}|  |¡\}}| j t|||d�¡ d S )Nrk   Frl   ©rm   ÚsrcÚdest)Úoperation_namer(   rF   Ú_format_src_destrc   Úputr   )r*   r_   rm   rx   ry   r-   r-   r.   rs     s   ÿz+BaseTransferRequestSubmitter._submit_dryrunc                 C   ó   d S ©Nr-   ©r*   ru   r_   r-   r-   r.   rp     ó   z8BaseTransferRequestSubmitter._add_additional_subscribersc                 C   rg   )Nz_submit_transfer_request()rh   )r*   r_   rt   ru   r-   r-   r.   rr     ó   z5BaseTransferRequestSubmitter._submit_transfer_requestc                 C   s    |   ¡ D ]	}||ƒr dS qd S )NT)Ú_get_warning_handlers)r*   r_   Úwarning_handlerr-   r-   r.   rd     s
   ûÿz5BaseTransferRequestSubmitter._warn_and_signal_if_skipc                 C   s   g S r~   r-   ©r*   r-   r-   r.   r‚     s   z2BaseTransferRequestSubmitter._get_warning_handlersc                 C   s   | j  d¡o| j  d¡ S )NÚguess_mime_typeÚcontent_type)r(   rF   r„   r-   r-   r.   Ú_should_inject_content_type$  s   þz8BaseTransferRequestSubmitter._should_inject_content_typec                 C   sf   | j  d¡s1| ¡ s1t d|j|jf ¡ | j  d¡s/td|j d|j|jf ƒ}| j 	|¡ dS dS )NÚforce_glacier_transferz@Encountered glacier object s3://%s. Not performing %s on object.Úignore_glacier_warningsús3://zðObject is of storage class GLACIER. Unable to perform %s operations on GLACIER objects. You must restore the object to be able to perform the operation. See aws s3 %s help for additional parameter options to ignore or force these transfers.TF)
r(   rF   Úis_glacier_compatibler3   r4   rx   rz   r   rc   r|   )r*   r_   Úwarningr-   r-   r.   Ú_warn_glacier*  s"   
ÿÿ
úþ
z*BaseTransferRequestSubmitter._warn_glacierc                 C   sD   dt jj }t j |j¡ |¡}|r t|jdƒ}| j |¡ dS dS )Nz..z#File references a parent directory.TF)	ÚosÚpathÚsepÚnormpathÚcompare_keyÚ
startswithr   rc   r|   )r*   r_   Úparent_prefixÚescapes_cwdrŒ   r-   r-   r.   Ú_warn_parent_reference?  s   ÿÿz3BaseTransferRequestSubmitter._warn_parent_referencec                 C   rg   )zAReturns formatted versions of a fileinfos source and destination.r{   rh   rj   r-   r-   r.   r{   L  s   z-BaseTransferRequestSubmitter._format_src_destc                 C   s   t |ƒS r~   )r   ©r*   r�   r-   r-   r.   Ú_format_local_pathP  r�   z/BaseTransferRequestSubmitter._format_local_pathc                 C   s   |  d¡r|S d| S )NrŠ   )r“   r—   r-   r-   r.   Ú_format_s3_pathS  s   
z,BaseTransferRequestSubmitter._format_s3_path)rI   rJ   rK   ro   rq   r/   rZ   rY   re   rs   rp   rr   rd   r‚   r‡   r�   r–   r{   r˜   r™   r-   r-   r-   r.   rb   ¸   s$    	rb   c                   @   sN   e Zd ZejZeZdd„ Zdd„ Z	dd„ Z
dd„ Zd	d
„ Zdd„ Zdd„ ZdS )rP   c                 C   ó
   |j dkS )NÚupload©rz   rj   r-   r-   r.   rY   ]  ó   
z!UploadRequestSubmitter.can_submitc                 C   sF   |  t|jƒ¡ |  ¡ r|  tƒ ¡ | j dd¡r!|  tƒ ¡ d S d S ©Nrk   F)rG   r   Úsizer‡   r   r(   rF   r"   r   r-   r-   r.   rp   `  s   ÿz2UploadRequestSubmitter._add_additional_subscribersc                 C   ó.   t |jƒ\}}|  |¡}| jj|||||d�S ©N)ÚfileobjÚbucketÚkeyrt   ru   )r   ry   Ú_get_fileinrL   r›   )r*   r_   rt   ru   r£   r¤   Úfileinr-   r-   r.   rr   g  ó   
þz/UploadRequestSubmitter._submit_transfer_requestc                 C   ó   |j S r~   )rx   rj   r-   r-   r.   r¥   o  ó   z"UploadRequestSubmitter._get_fileinc                 C   ó   | j gS r~   )Ú_warn_if_too_larger„   r-   r-   r.   r‚   r  r�   z,UploadRequestSubmitter._get_warning_handlersc                 C   sT   t |dƒr&|jtkr(t|jƒ}d|ttƒf }t||dd�}| j |¡ d S d S d S )NrŸ   z&File %s exceeds s3 upload limit of %s.F)Ú	skip_file)	ÚgetattrrŸ   r   r   rx   r   r   rc   r|   )r*   r_   Ú	file_pathÚwarning_messagerŒ   r-   r-   r.   r«   u  s   
ÿÿÿùz)UploadRequestSubmitter._warn_if_too_largec                 C   ó    |   |j¡}|  |j¡}||fS r~   )r˜   rx   r™   ry   ©r*   r_   rx   ry   r-   r-   r.   r{     ó   z'UploadRequestSubmitter._format_src_destN)rI   rJ   rK   r   Úmap_put_object_paramsro   r
   rq   rY   rp   rr   r¥   r‚   r«   r{   r-   r-   r-   r.   rP   Y  s    
rP   c                   @   sF   e Zd ZejZeZdd„ Zdd„ Z	dd„ Z
dd„ Zd	d
„ Zdd„ ZdS )rQ   c                 C   rš   )NÚdownloadrœ   rj   r-   r-   r.   rY   ‰  r�   z#DownloadRequestSubmitter.can_submitc                 C   sV   |  t|jƒ¡ |  tƒ ¡ |  t|j| jƒ¡ | j dd¡r)|  t	|j
ƒ¡ d S d S rž   )rG   r   rŸ   r!   r    Úlast_updaterc   r(   rF   r#   Úsource_clientr   r-   r-   r.   rp   Œ  s   ÿ
ÿÿz4DownloadRequestSubmitter._add_additional_subscribersc                 C   r    r¡   )r   rx   Ú_get_fileoutrL   r´   )r*   r_   rt   ru   r£   r¤   Úfileoutr-   r-   r.   rr   •  r§   z1DownloadRequestSubmitter._submit_transfer_requestc                 C   r¨   r~   )ry   rj   r-   r-   r.   r·   �  r©   z%DownloadRequestSubmitter._get_fileoutc                 C   s   | j | jgS r~   )r�   r–   r„   r-   r-   r.   r‚      s   z.DownloadRequestSubmitter._get_warning_handlersc                 C   r°   r~   )r™   rx   r˜   ry   r±   r-   r-   r.   r{   £  r²   z)DownloadRequestSubmitter._format_src_destN)rI   rJ   rK   r   Úmap_get_object_paramsro   r   rq   rY   rp   rr   r·   r‚   r{   r-   r-   r-   r.   rQ   …  s    	rQ   c                   @   s>   e Zd ZejZeZdd„ Zdd„ Z	dd„ Z
dd„ Zd	d
„ ZdS )rR   c                 C   rš   )NÚcopyrœ   rj   r-   r-   r.   rY   ­  r�   zCopyRequestSubmitter.can_submitc                 C   sJ   |  t|jƒ¡ |  ¡ r|  tƒ ¡ | j dd¡r#|  t|jƒ¡ d S d S rž   )	rG   r   rŸ   r‡   r   r(   rF   r$   r¶   r   r-   r-   r.   rp   °  s   
ÿÿz0CopyRequestSubmitter._add_additional_subscribersc           	      C   s@   t |jƒ\}}t |jƒ\}}||dœ}| jj||||||jd�S )N)ÚBucketÚKey)r£   r¤   Úcopy_sourcert   ru   r¶   )r   ry   rx   rL   rº   r¶   )	r*   r_   rt   ru   r£   r¤   Úsource_bucketÚ
source_keyr½   r-   r-   r.   rr   ¸  s   
ýz-CopyRequestSubmitter._submit_transfer_requestc                 C   rª   r~   )r�   r„   r-   r-   r.   r‚   Â  r�   z*CopyRequestSubmitter._get_warning_handlersc                 C   s    |   |j¡}|   |j¡}||fS r~   )r™   rx   ry   r±   r-   r-   r.   r{   Å  r²   z%CopyRequestSubmitter._format_src_destN)rI   rJ   rK   r   Úmap_copy_object_paramsro   r   rq   rY   rp   rr   r‚   r{   r-   r-   r-   r.   rR   ©  s    
rR   c                   @   ó0   e Zd ZeZdd„ Zdd„ Zdd„ Zdd„ Zd	S )
rN   c                 C   ó   |j dko
| j d¡S )Nr›   rD   ©rz   r(   rF   rj   r-   r-   r.   rY   Î  ó   

þz'UploadStreamRequestSubmitter.can_submitc                 C   s0   | j  dd ¡}|d ur| tt|ƒƒ¡ d S d S )NÚexpected_size)r(   rF   rG   r   Úint)r*   ru   r_   rÅ   r-   r-   r.   rp   Ô  s   ÿz8UploadStreamRequestSubmitter._add_additional_subscribersc                 C   s   t ƒ }t|ƒS r~   )r%   r   )r*   r_   Úbinary_stdinr-   r-   r.   r¥   Ù  s   z(UploadStreamRequestSubmitter._get_fileinc                 C   ó   dS ©Nú-r-   r—   r-   r-   r.   r˜   Ý  r€   z/UploadStreamRequestSubmitter._format_local_pathN)	rI   rJ   rK   r   rq   rY   rp   r¥   r˜   r-   r-   r-   r.   rN   Ë  s    rN   c                   @   rÁ   )
rO   c                 C   rÂ   )Nr´   rD   rÃ   rj   r-   r-   r.   rY   ä  rÄ   z)DownloadStreamRequestSubmitter.can_submitc                 C   r}   r~   r-   r   r-   r-   r.   rp   ê  r€   z:DownloadStreamRequestSubmitter._add_additional_subscribersc                 C   s   t ƒ S r~   r   rj   r-   r-   r.   r·   í  r©   z+DownloadStreamRequestSubmitter._get_fileoutc                 C   rÈ   rÉ   r-   r—   r-   r-   r.   r˜   ð  r€   z1DownloadStreamRequestSubmitter._format_local_pathN)	rI   rJ   rK   r   rq   rY   rp   r·   r˜   r-   r-   r-   r.   rO   á  s    rO   c                   @   s.   e Zd ZejZeZdd„ Zdd„ Z	dd„ Z
dS )rS   c                 C   ó   |j dko	|jdkS )NÚdeleteÚs3©rz   Úsrc_typerj   r-   r-   r.   rY   ø  ó   
ÿz!DeleteRequestSubmitter.can_submitc                 C   s"   t |jƒ\}}| jj||||d�S )N)r£   r¤   rt   ru   )r   rx   rL   rÌ   )r*   r_   rt   ru   r£   r¤   r-   r-   r.   rr   ü  s
   þz/DeleteRequestSubmitter._submit_transfer_requestc                 C   ó   |   |j¡d fS r~   )r™   rx   rj   r-   r-   r.   r{     ó   z'DeleteRequestSubmitter._format_src_destN)rI   rJ   rK   r   Úmap_delete_object_paramsro   r   rq   rY   rr   r{   r-   r-   r-   r.   rS   ô  s    rS   c                   @   s,   e Zd ZdZdZdd„ Zdd„ Zdd„ ZdS )rT   Nc                 C   rË   )NrÌ   ÚlocalrÎ   rj   r-   r-   r.   rY   
  rÐ   z&LocalDeleteRequestSubmitter.can_submitc              
   C   s°   |   |¡\}}d||dœ}zDz"| j tdddi|¤Ž¡ t |j¡ | j tdi |¤Ž¡ W W dS  tyQ } z| j t	dd|i|¤Ž¡ W Y d }~W dS d }~ww    Y dS )NrÌ   rw   Útotal_transfer_sizer   Ú	exceptionTr-   )
r{   rc   r|   r   rŽ   Úremoverx   r   Ú	Exceptionr   )r*   r_   rt   ru   rx   ry   rv   Úer-   r-   r.   rr     s,   ýÿÿûÿ€ûz4LocalDeleteRequestSubmitter._submit_transfer_requestc                 C   rÑ   r~   )r˜   rx   rj   r-   r-   r.   r{   -  rÒ   z,LocalDeleteRequestSubmitter._format_src_dest)rI   rJ   rK   ro   rq   rY   rr   r{   r-   r-   r-   r.   rT     s    rT   )8ÚloggingrŽ   Ús3transfer.managerr   Úawscli.customizations.s3.utilsr   r   r   r   r   r   Ú'awscli.customizations.s3.transferconfigr	   Ú awscli.customizations.s3.resultsr
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   Úawscli.compatr%   Ú	getLoggerrI   r3   Úobjectr&   r8   rb   rP   rQ   rR   rN   rO   rS   rT   r-   r-   r-   r.   Ú<module>   sX    
B? ",$"