o y`)V@sddlZddlmZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z ddlZddlmZmZmZddlmZmZmZdd lmZmZdd lmZmZdd lmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%ddl&m'Z'm(Z(e)e*Z+GdddZ,ddde'dde(ddfddZ-Gddde.Z/Gddde!Z0Gddde Z1GdddZ2Gddde2Z3Gd d!d!eZ4Gd"d#d#Z5Gd$d%d%Z6Gd&d'd'Z7Gd(d)d)Z8dS)*N)BytesIO)UNSIGNED)Config)urlsplit)NoCredentialsError)S3Client S3RequestTypeS3RequestTlsMode)ClientBootstrapDefaultHostResolverEventLoopGroup)ClientTlsContextTlsContextOptions)AwsCredentialsProviderAwsCredentials)TransferNotDoneError)BaseTransferFutureBaseTransferMeta)CallArgsOSUtils get_callbacks)GBMBc@s$eZdZddZddZddZdS)CRTCredentialProviderAdaptercCs||_d|_t|_dSN)_botocore_credential_provider_loaded_credentials threadingLock_lock)selfbotocore_credential_providerr"0/usr/lib/python3/dist-packages/s3transfer/crt.py__init__'sz%CRTCredentialProviderAdapter.__init__cCs|}t|j|j|jSr)_get_credentialsget_frozen_credentialsr access_key secret_keytoken)r credentialsr"r"r#__call__,s z%CRTCredentialProviderAdapter.__call__cCsX|j|jdur|j}|durt||_|jWdS1s%wYdSr)rrrload_credentialsr)r loaded_credsr"r"r#r%1s $z-CRTCredentialProviderAdapter._get_credentialsN)__name__ __module__ __qualname__r$r+r%r"r"r"r#r&s rTc Cst|}t|}t||} d} d} |rtjntj} |dur4t} |r)| j|dnd| _t | }| } |r?t |}t |} |dt}t| || || | |dS)a :type region: str :param region: The region used for signing :type botocore_credential_provider: Optional[botocore.credentials.CredentialResolver] :param botocore_credential_provider: Provide credentials for CRT to sign the request if not set, the request will not be signed :type num_threads: Optional[int] :param num_threads: Number of worker threads generated. Default is the number of processors in the machine. :type target_throughput: Optional[int] :param target_throughput: Throughput target in Bytes. Default is 0.625 GB/s (which translates to 5 Gb/s). :type part_size: Optional[int] :param part_size: Size, in Bytes, of parts that files will be downloaded or uploaded in. :type use_ssl: boolean :param use_ssl: Whether or not to use SSL. By default, SSL is used. Note that not all services support non-ssl connections. :type verify: Optional[boolean/string] :param verify: Whether or not to verify SSL certificates. By default SSL certificates are verified. You can provide the following values: * False - do not validate SSL certificates. SSL will still be used (unless use_ssl is False), but SSL certificates will not be verified. * path/to/cert/bundle.pem - A filename of the CA cert bundle to use. Specify this argument if you want to use a custom CA cert bundle instead of the default one on your system. N) ca_filepathFr2) bootstrapregioncredential_provider part_sizetls_modetls_connection_optionsthroughput_target_gbps)r r r r ENABLEDDISABLEDr&override_default_trust_store_from_path verify_peerr new_connection_optionsrr new_delegaterr)r5r! num_threadstarget_throughputr7use_sslverifyevent_loop_group host_resolverr4providerr9r8tls_ctx_optionsclient_tls_optioncredentails_provider_adapter target_gbpsr"r"r#create_s3_crt_client<sD-   rLc@seZdZdddZddZddZ  ddd Z  dd d Z  dd d ZdddZ ddZ ddZ ddZ dddZ ddZddZdS) CRTTransferManagerNcCs@|durt|_||_t||j|_g|_td|_d|_ dS)aFA transfer manager interface for Amazon S3 on CRT s3 client. :type crt_s3_client: awscrt.s3.S3Client :param crt_s3_client: The CRT s3 client, handling all the HTTP requests and functions under then hood :type crt_request_serializer: s3transfer.crt.BaseCRTRequestSerializer :param crt_request_serializer: Serializer, generates unsigned crt HTTP request. :type osutil: s3transfer.utils.OSUtils :param osutil: OSUtils object to use for os-related behavior when using with transfer manager. Nr) r_osutil_crt_s3_clientS3ClientArgsCreator_s3_args_creator_future_coordinatorsr Semaphore _semaphore _id_counter)r crt_s3_clientcrt_request_serializerosutilr"r"r#r$s  zCRTTransferManager.__init__cCs|Srr"r r"r"r# __enter__szCRTTransferManager.__enter__cGsd}|rd}||dS)NFT _shutdown)r exc_type exc_valueargscancelr"r"r#__exit__szCRTTransferManager.__exit__cCs6|duri}|dur i}t|||||d}|d|S)Nbucketkeyfileobj extra_args subscribers get_objectr_submit_transfer)r rdrerfrgrhcallargsr"r"r#downloads zCRTTransferManager.downloadcCs6|duri}|dur i}t|||||d}|d|S)Nrc put_objectrj)r rfrdrergrhrlr"r"r#uploads zCRTTransferManager.uploadcCs4|duri}|dur i}t||||d}|d|S)N)rdrergrh delete_objectrj)r rdrergrhrlr"r"r#deletes zCRTTransferManager.deleteFcCs||dSrr\r rar"r"r#shutdownzCRTTransferManager.shutdowncCs |jD] }|s |qdSr)rSdonerar coordinatorr"r"r#_cancel_transferss z$CRTTransferManager._cancel_transferscC|jD]}|qdSr)rSresultrvr"r"r#_finish_transfers  z$CRTTransferManager._finish_transferscCryr)rS%wait_until_on_done_callbacks_completervr"r"r#_wait_transfers_doner|z'CRTTransferManager._wait_transfers_donecCsz|r|z1z|Wnty|Ynty"YnwW|dSW|dSW|dS|wr)rxr{KeyboardInterrupt Exceptionr~rrr"r"r#r]s      zCRTTransferManager._shutdowncK|jdSr)rUreleaser kwargsr"r"r#_release_semaphorertz%CRTTransferManager._release_semaphorec Cs|jg}t|jd}t|j||d}td i|}t|}||z$|j|j |d}||j |||||} |j j d i| } Wn&tym} z|| d|j j |d|d} | | dWYd} ~ n d} ~ ww|| |j||jd7_|S) N) transfer_id)metarwqueuedTru)after_subscribers)errorr")rCRTTransferCoordinatorrVCRTTransferMetaCRTTransferFutureAfterDoneHandlerappendrUacquirerRget_crt_callbackget_make_request_argsrP make_requestr set_exceptionset_s3_requestrS) r request_type call_argson_done_after_callsrw componentsfuture afterdone on_queued crt_callargscrt_s3_requesteon_doner"r"r#rks>       z#CRTTransferManager._submit_transferrNNF)r.r/r0r$r[rbrmrorqrsrxr{r~r]rrkr"r"r"r#rMs(        rMc@s>eZdZdZd ddZeddZeddZed d ZdS) rz*Holds metadata about the CRTTransferFutureNcCs||_||_i|_dSr) _transfer_id _call_args _user_context)r rrr"r"r#r$s zCRTTransferMeta.__init__cC|jSr)rrZr"r"r#rzCRTTransferMeta.call_argscCrr)rrZr"r"r#rrzCRTTransferMeta.transfer_idcCrr)rrZr"r"r# user_context rzCRTTransferMeta.user_contextr) r.r/r0__doc__r$propertyrrrr"r"r"r#rs   rc@sDeZdZdddZeddZddZddd Zd d Zd d Z dS)rNcCs ||_|dur t|_||_dS)aZThe future associated to a submitted transfer request via CRT S3 client :type meta: s3transfer.crt.CRTTransferMeta :param meta: The metadata associated to the transfer future. :type coordinator: s3transfer.crt.CRTTransferCoordinator :param coordinator: The coordinator associated to the transfer future. N)_metar _coordinator)r rrwr"r"r#r$&s  zCRTTransferFuture.__init__cCrr)rrZr"r"r#r4rzCRTTransferFuture.metacCs |jSr)rrurZr"r"r#ru8 zCRTTransferFuture.donecC|j|dSr)rrzr timeoutr"r"r#rz;zCRTTransferFuture.resultcCrr)rrarZr"r"r#ra>rtzCRTTransferFuture.cancelcCs$|std|jj|dddS)z!Sets the exception on the future.z?set_exception can only be called once the transfer is complete.T)overrideN)rurrr)r exceptionr"r"r#rAs zCRTTransferFuture.set_exceptionrr) r.r/r0r$rrrurzrarr"r"r"r#r%s    rc@seZdZddZdS)BaseCRTRequestSerializercCstd)alSerialize CRT HTTP requests. :type transfer_type: string :param transfer_type: the type of transfer made, e.g 'put_object', 'get_object', 'delete_object' :type future: s3transfer.crt.CRTTransferFuture :rtype: awscrt.http.HttpRequest :returns: An unsigned HTTP request to be used for the CRT S3 client zserialize_http_request())NotImplementedError)r transfer_typerr"r"r#serialize_http_requestKs z/BaseCRTRequestSerializer.serialize_http_requestN)r.r/r0rr"r"r"r#rJs rc@sVeZdZdddZddZddZdd Zd d Zd d ZddZ ddZ ddZ dS)BotocoreCRTRequestSerializerNcCsp||_|dur i}||||jdi||_|jjjd|j|jjjd|j|jjjd|j dS)aSerialize CRT HTTP request using botocore logic It also takes into account configuration from both the session and any keyword arguments that could be passed to `Session.create_client()` when serializing the request. :type session: botocore.session.Session :type client_kwargs: Optional[Dict[str, str]]) :param client_kwargs: The kwargs for the botocore s3 client initialization. Nzrequest-created.s3.*zafter-call.s3.*zbefore-send.s3.*r") _session_resolve_client_config create_client_clientreventsregister_capture_http_request+_change_response_to_serialized_http_request_make_fake_http_response)r session client_kwargsr"r"r#r$[s     z%BotocoreCRTRequestSerializer.__init__cCsPd}|r |}d|vr|d}ttd}|r||}||d<d|d<dS)Nconfig)signature_versions3 service_name)get_default_client_configrrmerge)r rruser_provided_config client_configr"r"r#rts   z3BotocoreCRTRequestSerializer._resolve_client_configc Cst|j}|j}|jrd||jf}g}|jD]\}}t|tr*|||fq||t|dfqt j |}d}|j rOt |j drJ|j }nt|j }t j j|j|||d} | S)Nz%s?%szutf-8seek)methodpathheaders body_stream)rurlrqueryritems isinstancestrrawscrthttp HttpHeadersbodyhasattrr HttpRequestr) r aws_request url_partscrt_path headers_listnamevalue crt_headerscrt_body_stream crt_requestr"r"r#_crt_request_from_aws_requests,     z:BotocoreCRTRequestSerializer._crt_request_from_aws_requestcCsT||}|jddurt|j}|jd|j|jddur(|jd|S)Nhostz Content-MD5)rrgetrrsetnetlocremove)r botocore_http_requestrrr"r"r#_convert_to_crt_http_requests  z9BotocoreCRTRequestSerializer._convert_to_crt_http_requestcKs||jd<dS)N http_request)contextr requestrr"r"r#rrtz2BotocoreCRTRequestSerializer._capture_http_requestcKs|d}||d<dS)Nr HTTPRequest)prepare)r rparsedrrr"r"r#rszHBotocoreCRTRequestSerializer._change_response_to_serialized_http_requestcKstjdditdS)N)botocore awsrequest AWSResponseFakeRawResponserr"r"r#rs z5BotocoreCRTRequestSerializer._make_fake_http_responsecCs&t|j|d|j|jd|jdS)N)BucketKeyrr")getattrrrdrerg)r client_methodrr"r"r#_get_botocore_http_requests z7BotocoreCRTRequestSerializer._get_botocore_http_requestcCs|||jj}||}|Sr)rrrr)r rrrrr"r"r#rs  z3BotocoreCRTRequestSerializer.serialize_http_requestr) r.r/r0r$rrrrrrrrr"r"r"r#rZs    rc@seZdZdddZdS)rNccs ||}|s dS|Vqr)read)r amtdecode_contentchunkr"r"r#streams zFakeRawResponse.stream)rN)r.r/r0rr"r"r"r#rsrc@sdeZdZdZdddZeddZddZdd d Zdd d Z ddZ dddZ ddZ ddZ dS)rz-A helper class for managing CRTTransferFutureNcCs0||_||_t|_d|_d|_t|_dSr) r _s3_requestrrr _exception _crt_futureEvent _done_event)r r s3_requestr"r"r#r$s  zCRTTransferCoordinator.__init__cCrr)rrZr"r"r#r rz!CRTTransferCoordinator.s3_requestcCrr)r rrZr"r"r#set_done_callbacks_completertz2CRTTransferCoordinator.set_done_callbacks_completecCrr)r waitrr"r"r#r}rz|j}|j |} | |j d<|j ||} | |||||d| |||ddS)N ContentLengthruprogress)rtype recv_filepath send_filepathr on_progress)rrupperDEFAULT GET_OBJECTrfrget_temp_filenameRenameTempFileHandlerr PUT_OBJECT get_file_sizergrrr)r rrrwrrrrs3_meta_request_typeon_done_before_callsfinal_filepathfile_ondone_calldata_lenrr"r"r#rs@        z)S3ClientArgsCreator.get_make_request_argsNcsfdd}|S)Ncsfg}dur |7}|t7}dur|7}|D]}dkr)||ddq||i|qdS)Nrr)bytes_transferred)r)r`rcallbacks_listcallbackrbefore_subscribers callback_typerr"r#invoke_all_callbacks.szBS3ClientArgsCreator.get_crt_callback..invoke_all_callbacksr")r rr)r(rr*r"r'r#r+sz$S3ClientArgsCreator.get_crt_callbackr)r.r/r0r$rrr"r"r"r#rQs $rQc@eZdZddZddZdS)rcCs||_||_||_||_dSr)r_final_filename_temp_filenamerO)r rwfinal_filename temp_filenamerYr"r"r#r$Bs zRenameTempFileHandler.__init__c Ksx|d}|r|j|jdSz |j|j|jWdSty;}z|j|j|j|WYd}~dSd}~ww)Nr)rO remove_filer- rename_filer,rrr)r rrrr"r"r#r+Hs zRenameTempFileHandler.__call__Nr.r/r0r$r+r"r"r"r#rAs rc@r+)rcCs ||_dSr)rrvr"r"r#r$WrzAfterDoneHandler.__init__cKrr)rr rr"r"r#r+ZrtzAfterDoneHandler.__call__Nr2r"r"r"r#rVs r)9loggingiorrbotocore.awsrequestrbotocore.sessionrbotocore.configrbotocore.compatrbotocore.exceptionsr awscrt.httpr awscrt.s3rrr awscrt.ior r r r r awscrt.authrrs3transfer.exceptionsrs3transfer.futuresrrs3transfer.utilsrrrs3transfer.constantsrr getLoggerr.loggerrrLobjectrMrrrrrrrQrrr"r"r"r#sL         O%k 5>