o lh@s dZddlZddlZddlZddlZddlZddlmZddlm Z ddlm Z ddl m Z ddl mZddlZddlZddlmZdd lmZmZdd lmZdd lmZmZdd lmZdd lmZmZmZmZmZm Z ddl!m"Z"m#Z#zddl$Z Wn e%yddl Z Ynwzddl&m'Z'Wne%yddl'm'Z'Ynwzddl(Z(Wn e%ydZ(Ynwze)Wn e*ye+Z)Ynwze,Wne*yddl-m,Z,Ynwzddl.m/Z/Wn e%ydZ/Ynwzddl0Z0Wn e%ydZ0Ynwddl1m2Z2ddl1m3Z3ddl1m4Z4ddl1m5Z5ddl1m6Z6ddl1m7Z7ddl8m9Z9m:Z:ddl8m;Z;mZ>ddl8m?Z?m@Z@mAZAmBZBddlCmDZDddlCmEZEddlCmFZFdd lCmGZGdgeHe@IZJeJd!d"e@D7ZJd#d$gZKeKd%gZLeMe?d&reJNe?Od'e@eAZPd(d)ZQd*d+ZRd,d-ZSGd.d/d/eTZUd'd1d2ZVd3d4ZWd(d5d6ZXd7d8ZYd9d:ZZd;d<Z[d=d>Z\ed?eJed@gdAedBgdCdDdEZ]ed?eJdFdGZ^dHdIZ_eedJgdKdLdMZ`eed?gdNdOdPZadQdRZbedSe@edTe@dUdVZcdWdXZddYdZZed[d\Zfed@gd]d^d_Zged?eLd`daZhedbdcZied?eLdddeZjeed?eLdfdgZkeed?eKdhdiZldjdkZmed?e@edldgdmfdngdofgdpdqZneed?eLdrdsZodtduZpeed?eKdvdwZqdxdyZrdzd{Zsd|d}Zteed?eLd~dZuddZvddZwGddde2ZxddZyed?eJddZzddZ{ddZ|ddZ}eLddZ~e~dd"edDeed?e~ddZGddde2ZddZddZddZddZGdddejj6Ze;deeed?gdddZeed@gded?eLddZddZeedeLedeLddZeddZddZddZedgdddZedgdddZeddZeeed?eKddZdZeed?eKddZdZdZdZdZeed?eKejddƅdkse?durgnd'gedeeegedgdʢdd̄Zdjejejejd΍ZeddЄZdjejejejd΍ZeddӄZddՄZddׄZeeed?eKddلZddۄZeedd݄Zdd߄Zed?eJed@gdddZeeed?eKddZed?gdddZddZeed?eKddZddZddZddZddZddZd)ddZeed?d$d%gddZeee0duddddZd*ddZed?d$d%gddZddZed?dd$gee/duddddZee0dudddd Zd d Zd d ZddZddZddZeeeejdkdded@gdddZeeeejdkddedgded@dƐdgddZeed@dƐdgedgdd d!Zeeed?d#d%e4e3gd"d#Zeed@gd$d%d&ZdS(+z Test the parallel module. N)format_exceptionsqrt)sleep) PicklingError TimeoutErrorparallel)dumpload)get_reusable_executor)np with_numpy)with_multiprocessing) parametrizeraisescheck_subprocess_callskipifSkipTestwarns) PY3_OR_LATERPY27)Queue)reload) parallel_sum)SequentialBackend)ThreadingBackend)MultiprocessingBackend)ParallelBackendBase) LokyBackend) SafeFunction)Paralleldelayed)register_parallel_backendparallel_backend)effective_n_jobs cpu_count)mpBACKENDSDEFAULT_BACKENDEXTERNAL_BACKENDS)JoblibException)TransportableException)JoblibValueError)WorkerInterruptcCsg|]}t|qS)r*).0 backend_strr1r1K/var/www/html/env/lib/python3.10/site-packages/joblib/test/test_parallel.py Xr5multiprocessingloky threading get_contextspawncCst|dt|ddS)N_pool_workers)getattrbackendr1r1r4 get_workerscsrAcCs||SNr1)xyr1r1r4divisiongrEcC|dSNr1rCr1r1r4squarekrFrKc@seZdZdZddZdS)MyExceptionWithFinickyInitz1An exception class with non trivial __init__ cCdSrBr1)selfabcdr1r1r4__init__rz#MyExceptionWithFinickyInit.__init__N__name__ __module__ __qualname____doc__rSr1r1r1r4rLos rLFcCs"|dkr|r tddddt|S)NrOrPrQrR)rL ValueError)rCcustom_exceptionr1r1r4exception_raiservs r]cCstdt)Ng?)timerKeyboardInterruptrJr1r1r4interrupt_raiser}s r`cCs|d||S)zO A module-level function so that it can be spawn with multiprocessing. rIr1)rCrDzr1r1r4fsrbcCsttdSNr)typer get_active_backendr1r1r1r4_active_backend_typesrfcCst||dddtdDS)Nn_jobsr@cs|] }tt|VqdSrBr$rKr2ir1r1r4   z parallel_func..r#range) inner_n_jobsr@r1r1r4 parallel_funcsrscCtdksJdSrc)r(r1r1r1r4test_cpu_countrucCrtrc)r'r1r1r1r4test_effective_n_jobsrvrwr@rh)rrIverbose)rI dcCs:ddtdDt|||dddtdDksJdS)NcSg|]}t|qSr1rKr2rCr1r1r4r5z(test_simple_parallel..)rhr@rzcsrirBrjrr1r1r4rmrnz'test_simple_parallel..rqr#)r@rhrzr1r1r4test_simple_parallels rcCs|jtdddtd}td|dddtdD}|gd ks&JWdn1s0wYd d |D}t|d ksDJdS) Nname!some_new_name_for_the_main_thread)targetrvaluerIrgcsrirBrjrr1r1r4rmrnz6test_main_thread_renamed_no_warning..ro)rrcSsg|] }dt|jvr|qS)zworker timeout)strmessage)r2wr1r1r4r5z7test_main_thread_renamed_no_warning..r)setattrr9current_threadrr#rqlen)r@ monkeypatchwarninforesultsr1r1r4#test_main_thread_renamed_no_warnings  rcCsptd}t||dWdn1swY|r.t|dkr,d|djjdvSdSt|dks6JdS)N)r@rrrzbacked parallel loops cannotFT)rrsrrargs)r@rrexpectedrecordsr1r1r4_assert_warning_nesteds  rz%parent_backend,child_backend,expected))r8r7T)r8r8F)r7r7T)r7r8T)r9r7T)r9r8Tcsntd|dfddtdDtd|dfddtdD}|dkr/t|s-JdSt|s5JdS)NrIrgc3s"|] }ttdddVqdS)rFr@rrrNr$rr2_ child_backendr1r4rm z0test_nested_parallel_warnings..rc3s"|] }ttddVqdS)rIrNrrrrr1r4rmrr9)r#rqanyall)parent_backendrrresr1rr4test_nested_parallel_warningss r)r8r7r9cCs>dg}dd}tj||fd}|||dsJdS)NFcSsdtd}tddddtdDWdn1swYtt|t|dk|d<dS)NrIrhcs|] }ttdVqdS)皙?Nr$rrr1r1r4rmrnzPtest_background_thread_parallelism..background_thread..rr)rr#rqprintr)is_run_parallelrr1r1r4background_threads   z=test_background_thread_parallelism..background_thread)rrr)r9Threadstartjoin)r@rrtr1r1r4"test_background_thread_parallelisms rcCs"td|dddtdDdS)NrIrgcsr){Gz?Nrjrr1r1r4rmrnznested_loop..rpr?r1r1r4 nested_loops rrrcs&td|dfddtdDdS)NrIrgc3|] }ttVqdSrB)r$rrrr1r4rmrnz#test_nested_loop..rp)rrr1rr4test_nested_loops rcCstrBr[r?r1r1r4raise_exception rTrc Cstt5tddd}|ttdttdgWdn1s$wYWddSWddS1s.N)rr#rqfullr1r1rr4test_mutate_input_with_threadss r)rrIrocCs6td}dd|Dt|ddd|DksJdS)z.Check the keyword argument processing of pmap. cSg|]}t|ddqSrrDrbrr1r1r4r5#z(test_parallel_kwargs..rcs |] }tt|ddVqdSrrNr$rbrr1r1r4rm$z'test_parallel_kwargs..Nr)rhlstr1r1r4test_parallel_kwargss  rcCstd}dd|D}td|d4}|j}||dd|Dks"J||dd|Dks/Jtdur>t|t|jus>JWdn1sHwYtdurZt|jdusZJ||d d|DksgJtdurtt|jdusvJdSdS) NrcSrrrrr1r1r4r5*rz4test_parallel_as_context_manager..rrgcsrrrrr1r1r4rm3rz3test_parallel_as_context_manager..csrrrrr1r1r4rm4rcsrrrrr1r1r4rm@r)rqr#_backendr)rA)r@rrpmanaged_backendr1r1r4 test_parallel_as_context_manager's"rcsbGdddtttddtddfddtd DWd d S1s*wYd S) ze Check that pmap captures the errors when it is passed an object that cannot be pickled. c@seZdZddZdS)z1test_parallel_pickling..UnpicklableObjectcStd)N123) RuntimeErrorrNr1r1r4 __reduce__KrFz.UnpicklableObject.__reduce__N)rVrWrXrr1r1r1r4UnpicklableObjectJs rzthe task to sendmatchrIrc3s|] }ttVqdSrBr$idrrr1r4rmOz)test_parallel_pickling..rN)objectrrr#rqr1r1rr4test_parallel_picklingEs""rcCs0ttd|ddddtdDdksJdS)NrIrrhr@timeoutcsr)gMbP?Nrrr1r1r4rmUrnz0test_parallel_timeout_success..)rr#rqr?r1r1r4test_parallel_timeout_successRs  rcCsNtttd|ddddtdDWddS1s wYdS)NrIrrcsr)rNrrr1r1r4rm^rnz-test_parallel_timeout_fail..r)rrr#rqr?r1r1r4test_parallel_timeout_failYs  "rc CsJtdur tttd|dddtddDWdn1s$wYtttd|ddddDWdn1sEwYtd|d}t|jdusZJt|j}tt|ddtddDWdn1szwYt|jdusJt|j|usJd dtd D|d d td DksJt|j}tt|d ddDWdn1swYt|jdusJt|j|usJddtd D|dd td DksJWdn1swYt|jdus Jn!tt tdddddDWdn 1s'wYtttddddtddDWdn 1sKwYtt tddddd tdDWdn 1sowYztdddd tddDWtdt y}zt |t rJWYd}~dSd}~ww)NrIrgcSg|] \}}tt||qSr1r$rEr2rCrDr1r1r4r5jz&test_error_capture..)rr)rrcSg|]}tt|qSr1r$r`rr1r1r4r5nrcSrr1rrr1r1r4r5vrcSrrrrr1r1r4r5rrcsrrrrr1r1r4rmrz%test_error_capture..cSrr1rrr1r1r4r5rcSrrrrr1r1r4r5rcsrrrrr1r1r4rmrrcSrr1rrr1r1r4r5rcSrr1rrr1r1r4r5rr)rhrzcsr)T)r\Nr$r]rkr1r1r4rmsrcss"|] \}}tt||VqdSrBrrr1r1r4rm z'The excepted error has not been raised.)r)rZeroDivisionErrorr#zipr0rArrqr_rL Exception isinstancer-r[)r@r original_workersexr1r1r4test_error_capturebs~            !      rcCs|d|dS)Nz Consumed %s)append)queueitemr1r1r4consumerrvrzbatch_size, expected_queue) Produced 0 Consumed 0 Produced 1 Consumed 1 Produced 2 Consumed 2 Produced 3 Consumed 3 Produced 4 Consumed 4 Produced 5 Consumed 5r) rrrrrrrrrrrrcsTtfdd}td||dfdd|D|ks Jtdks(JdS) zC Test that with only one job, Parallel does act as a iterator. c3(tdD] }d||VqdSNz Produced %irqrrlrr1r4producer  z'test_dispatch_one_job..producerrrh batch_sizer@c3s|] }tt|VqdSrBr$rrrr1r4rmrz(test_dispatch_one_job.. N)listr#r)r@r expected_queuerr1rr4test_dispatch_one_jobs  rcst}|fdd}tddd|dfdd|Dt}|d d ks,J|d d d }|dks;J|d}||ksFJtdksNJd S)zW Check that using pre_dispatch Parallel does indeed dispatch items lazily. c3rrrrrr1r4rr z/test_dispatch_multiprocessing..producerrIrro)rhr  pre_dispatchr@c3|] }ttdVqdS)rNr rrr1r4rmrz0test_dispatch_multiprocessing..rrNrz Consumed anyrxrr )r)Managerrr#indexr)r@managerrqueue_contentsfirst_consumption_indexproduced_3_indexr1rr4test_dispatch_multiprocessings    rcCs\tdddd}|ddtdD|jdksJWddS1s'wYdS) NrIautor9r csrirBrrkr1r1r4rmz/test_batching_auto_threading..rr#rqrcompute_batch_size)rr1r1r4test_batching_auto_threadings"rcCs\tdd|d}|ddtdD|jdksJWddS1s'wYdS)NrIrr csrirBrrkr1r1r4rmrz2test_batching_auto_subprocesses..rrr)r@rr1r1r4test_batching_auto_subprocessess"r cCsNtttddddddtdDWddS1s wYdS) zCMake sure that exception raised during dispatch are indeed capturedrIr)rhrrzcsrirBrrkr1r1r4rmrnz*test_exception_dispatch..rN)rr[r#rqr1r1r1r4test_exception_dispatchs  "r"cC tddddtdDdS)NrIrcsrirBrr2jr1r1r4rm rnz(nested_function_inner..rrprr1r1r4nested_function_inner  r&cCr#)NrIrcsrirB)r$r&r$r1r1r4rmrnz(nested_function_outer..rrprr1r1r4nested_function_outerr'r(cCstr |dkr tdtt}td|dddtdDWdn1s(wYt|j|j|j }d |}d |vsAJd |vsGJd |vsMJt rZt|jtusXJdSt|jt uscJdS) aGEnsure errors for nested joblib cases gets propagated For Python 2.7, the TransportableException wrapping and unwrapping should preserve the traceback information of the inner function calls. For Python 3, we rely on the built-in __cause__ system that already report this kind of information to the user. r7zNNested parallel calls can deadlock with the python 2.7multiprocessing backend.rIrgcsrirB)r$r(rkr1r1r4rm$rnz1test_nested_exception_dispatch..rNr(r&r]) rrrr[r#rqrrdrtbrrr/)r@excinfo report_linesreportr1r1r4test_nested_exception_dispatchs       r.cCsBtjtj}|dd}|dd|}t|}t|dS)Nrz parallel.py/) r#rWsplitosseprr __import__r) joblib_pathmoduler1r1r4_reload_joblib;s     r6cCsdttjdds ttttdddddtdDWddS1s+wYdS) NJOBLIB_MULTIPROCESSINGrrIr)rhrcSsg|]}ttqSr1)r$r6rkr1r1r4r5Nr6z*test_multiple_spawning..r) intr1environgetrr ImportErrorr#rqr1r1r1r4test_multiple_spawningFs  "r<c@s$eZdZdZdddZd ddZdS) FakeParallelBackendz8Pretends to run concurrently while running sequentially.rNcKs|||_||_|SrB)r'rhr )rNrhr backend_argsr1r1r4 configureTs zFakeParallelBackend.configurecCs"|dkrttd|d}|S)Nrr)maxr)r(rNrhr1r1r4r'Ysz$FakeParallelBackend.effective_n_jobsr)r)rVrWrXrYr?r'r1r1r1r4r=Qs r=cCs8tttddWddS1swYdS)Nz unit-testingr?rr[r#r1r1r1r4test_invalid_backend_s  "rCcCsNtt}td|dWdn1swYdt|jvs%JdS)Nrrgz&n_jobs == 0 in Parallel has no meaning)rr[r#_initialize_backendrr)r@r+r1r1r4test_invalid_njobsds rEcCs<ztdtdtvs JtdtksJWtd=dStd=w)N test_backend)r%r=r*r1r1r1r4test_register_parallel_backendks   rGcCsTttksJztdtdddttksJWtt_ntt_wttks(JdS)Nr9T) make_default)rfDefaultBackendr%r*rr+r r1r1r1r4test_overwrite_default_backendts rJcCs^t|ddt\}}|dksJtddksJt}|jdks%J|dkr;t|tks1Jt|jtks:JnB|dkrQt|t ksGJt|jt ksPJn4|dkrgt|t ks]Jt|jt ksfJn.| drt|t kstJt|jt ksJWddSWddSWddSWddSWddS1swYdS)Nrorr7r8r9test_) r&r rer'r#rhrdrrr!r startswithr=) backend_nameactive_backend active_n_jobsrr1r1r4check_backend_context_managers8    "rPcCsg|]}d|qS)ztest_backend_%dr1rkr1r1r4r5rrocCsf|tvr |t|tttksJt|ttksJtdddddtDttks1JdS)NrIr9rgcss |] }|stt|VqdSrB)r$rP)r2rPr1r1r4rms   z/test_backend_context_manager..)r*setitemr=rfrIrPr# all_backends_for_context_manager)rr@r1r1r4test_backend_context_managersrSc@seZdZdZdddZdS)ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.NcCs|durtd||_dS)Nzparam should not be None)r[param)rNrUr1r1r4rSs z%ParameterizedParallelBackend.__init__rBrUr1r1r1r4rTsrTcCs|tdtttksJtdddd?t\}}t|tks$J|j dks+J|dks1Jt }|j dks;J|j |usBJ|ddt dD}Wdn1sWwY|dd t dDksiJttkspJdS) N param_backend*ro)rUrhcsrirBr$rrkr1r1r4rmrz=test_parameterized_backend_context_manager..rcSr}r1rrkr1r1r4r5rz>test_parameterized_backend_context_manager..)rQr*rTrfrIr&r rerdrUr#rhrrq)rrNrOrrr1r1r4*test_parameterized_backend_context_managers    rYcCsttksJttdddd?t\}}t|tksJ|jdks&J|dks,Jt}|j dks6J|j |us=J|ddt dD}Wdn1sRwY|ddt dDksdJttkskJdS) N+)rUrrcsrirBrXrkr1r1r4rmrzFtest_directly_parameterized_backend_context_manager..cSr}r1rrkr1r1r4r5rzGtest_directly_parameterized_backend_context_manager..) rfrIr&rTr rerdrUr#rhrrq)rNrOrrr1r1r43test_directly_parameterized_backend_context_managers   r[cCstdtS)Nr)rr1getpidr1r1r1r4sleep_and_return_pidsr]cCs<ttksJtdksJtddddtdDS)NrrIrcs|]}ttVqdSrB)r$r]rr1r1r4rmsz"get_nested_pids..)rfrr#_effective_n_jobsrqr1r1r1r4get_nested_pidss r`cs eZdZdZfddZZS) MyBackendz:Backend to test backward compatibility with older backendscstt|dSrc)superraget_nested_backendr __class__r1r4rcszMyBackend.get_nested_backend)rVrWrXrYrc __classcell__r1r1rdr4rasraback_compat_backend)r9r8r7rgcCsht|&tddddtdD}|D] }tt|dks!JqWddS1s-wYdS)NrIrcsr^rB)r$r`rr1r1r4rms  z6test_nested_backend_context_manager..rr)r&r#rqrset)r@ pid_groups pid_groupr1r1r4#test_nested_backend_context_managers "rk)rIrxNcs~ddtddfddtdDtdtddfddtdDWddS1s8wYdS) NcSs0tt|ks Jt|}t|ksJdSrB)rfr*r'r#r_)expected_backend_typeexpected_n_jobr1r1r4check_nested_backendsz?test_nested_backend_in_sequential..check_nested_backendrrc3s|] }tddVqdS)r8rNr$r)rnr1r4rm  z4test_nested_backend_in_sequential..rc3s|] }tVqdSrBrorr@rnrhr1r4rm#rp)r#rqr&)r@rhr1rqr4!test_nested_backend_in_sequential s  "rrcCs@t|\}}|j|ksJWddS1swYdSrB)r& nesting_level) inner_backendexpected_levelr@rhr1r1r4check_nesting_level)s"rv outer_backendrtcszt|dtd|dfddtdDtddtfddtdDWddS1s6wYdS) NrrIrgc3rrr$rvrrtr1r4rm5rpz-test_backend_nesting_level..rrc3rrrxrryr1r4rm;s)rvr#rqr&)rwrtr1ryr4test_backend_nesting_level.s  "rzcsddlGfdddt}td|ddtd"\}}tddfd d td D|jd ks5JWddS1s@wYdS) NrcseZdZdZjddZdS)z)test_retrieval_context..MyBackendrcss|jd7_dVdS)Nrrrr1r1r4retrieval_contextFs z;test_retrieval_context..MyBackend.retrieval_contextN)rVrWrXrlcontextmanagerr{r1) contextlibr1r4raCsra retrievalcSstddddt|DS)NrIrcsrirBrrkr1r1r4rmNrz>test_retrieval_context..nested_call..rp)nr1r1r4 nested_callMsz+test_retrieval_context..nested_callrIrc3s |] }tdd|VqdS)F check_pickleNrork)rr1r4rmQs  z)test_retrieval_context..rr)r}rr%r&r#rqrl)rabarr1)r}rr4test_retrieval_context?s "rcCstd}t|t|dS)Nfoobar)r-reprpickledumps)er1r1r4test_joblib_exceptionZsrcCstt}tr tt |ddWdn1swYn#tt }|ddWdn1s4wYt|jtsCJtt }tt  |dWddS1s[wYdS)NrrrC) r"rErrrr.rrunwrapr`r0) safe_divisionr+safe_interruptr1r1r4test_safe_functioncs      "rr )rrxgQ?cCs8ttt|dWddS1swYdS)Nr rBrr1r1r4test_invalid_batch_sizeus  "rz)n_tasks, n_jobs, pre_dispatch, batch_size) )rIrIrr)rIrIrhrrrIrhr)irIrhrr)rrrhr)r rhr)r  2 * n_jobsr)r rr)rr rrZ)rr rrcCsN|||d}ddt|D}tdi|ddt|D}||ks%JdS)N)rhrr cSr}r1r~rkr1r1r4r5rz0test_dispatch_race_condition..csrirBrjrkr1r1r4rmrz/test_dispatch_race_condition..r1r)n_tasksrhrr paramsrrr1r1r4test_dispatch_race_condition{s  rcCstddd}|jd}tjdkrA|}tjddpd}|dur9tj dkr1|d ks/JdS|d ks7JdS||ks?JdS|dusGJdS) NrIr7rgcontextrorJOBLIB_START_METHODr)win32r;fork) r# _backend_argsr:sys version_infoget_start_methodr1r9stripplatform)rr start_method env_methodr1r1r4test_default_mp_contexts    rcsp|dkrtjdkr tdtd}tjd}|ddt j t d|dfd d t dDdS) Nr7rzHmultiprocessing can cause BLAS freeze on old Python that relies on fork.r;rWirIrgc3s"|] }ttjjVqdSrB)r$rdotTrkrOr1r4rmrzAtest_no_blas_crash_or_freeze_with_subprocesses..) rrrr)r:rrandom RandomStaterandnrrr#rq)r@rngr1rr4.test_no_blas_crash_or_freeze_with_subprocessess     rafrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) cCs@|dkrtjdkrtdt|}ttjd|gddddS)Nr7rzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r stdout_regex)rrpytestskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatr executable)r@coder1r1r42test_parallel_with_interactively_defined_functionss    raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 rIr define_funccallable_position)r$rkwargsc Csv|dvr|tks tjdkrtdtj|||tj tj t j d}| d}| |ttj|jgddddS) N)r7r;rzNot picklable with pickle)rr@rjoblib_root_folderzunpicklable_func_script.pyrrr) SQUARE_MAINrrrr)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrr1pathdirnamejoblib__file__rwriterrstrpath)r@rrtmpdirr code_filer1r1r40test_parallel_with_unpicklable_functions_in_argss     raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )rcCs.|d}|tttj|jgddddS)Nz(joblib_interactively_defined_function.pyrr)rr)rr5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrrrrscriptr1r1r4Btest_parallel_with_interactively_defined_functions_default_backendPs    rauimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(n_jobs=2)( delayed(l.append)(i) for i in range(3) )) cCs0|d}|tttj|jgdddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\] LokyProcess)r stderr_regexr)rr9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrrrrr1r1r45test_parallel_with_interactively_defined_bound_methodws    rcCs"tg}tdd|gksJdSNrIr)iterr#)exhausted_iteratorr1r1r4%test_parallel_with_exhausted_iteratorsrcCs"t|tjs tdt||S)Nz#Expected np.memmap instance, got %r)rrmemmap TypeErrorrdcopyrr1r1r4 check_memmaps rcCsdd}tdd|ddd|dD}t||t|D] \}}tj||qtd d|dd d|dD}t||t|D] \}}tj||qBdS) Ncss*t|D] }tjdtjd|VqdS)Nrdtype)rqronesfloat32)rrlr1r1r4generate_arrayss zBtest_auto_memmap_on_arrays_from_generator..generate_arraysrIrrh max_nbytesr@csrirBr$rr2rOr1r1r4rmrnz.r|rcsrirBrrr1r1r4rmrn)r#rrrtestingassert_array_equal)r@rrresultrr1r1r4)test_auto_memmap_on_arrays_from_generatorsrcCs|SrBr1)argr1r1r4identityrTrcs|dj}tj}tj|ddtj|ddg}t||t|ddt ddfdd d D\}t d tj s;Jd j |ksDJtj ||dS) Nz test.mmapuint8rr) mmap_moderIrc3rrB)r$rrrr1r4rmrz.test_memmap_with_big_offset..)rr)rrmmapALLOCATIONGRANULARITYrzerosrr r r#rroffsetrr)rfnamesizeobjrr1rr4test_memmap_with_big_offsets   rcCstd}tddddtdDWdn1swYt|dks*J|d}t|jts6Jt|jdks?JdS)Nr)rcsrirBrjrkr1r1r4rmrzFtest_warning_about_timeout_not_supported_by_backend..2rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.)rr#rqrrr UserWarningr)rrr1r1r43test_warning_about_timeout_not_supported_by_backends  r)rrIryrxcCsdgdgd}tjddur|dkr|d7}ttt}t||ddd |DWdn1s8wYt|}|d ksIJdS) NrOrr|TRAVIS_OS_NAMErrgcss|] }ttj|VqdSrBr$r^rrkr1r1r4rmrz%test_abort_backend..)r1r9r:rrr^r#)rhr@delayst_startdtr1r1r4test_abort_backends   rcCs|j}tdd||d#}|ddtjdgdDtt|dks'JWdn1s1wYt|r=Jtdd|d}|d dtjdgdDt|r[JdS) NrIr)rhrr@ temp_foldercsrirBrrr1r1r4rmrz(test_memmapping_leaks..rrrcsrirBrrr1r1r4rmr)rr#rrrr1listdir)r@rrr1r1r4test_memmapping_leakss   r)Nr8r9cCs<td|dddtdD}|ddtdDksJdS)NrIrgcss |] }tdd|VqdS)cSrGrHr1rJr1r1r4sz2test_lambda_expression...Nrorkr1r1r4rms z)test_lambda_expression..rcSsg|]}|dqS)rIr1rkr1r1r4r5rz*test_lambda_expression..rp)r@rr1r1r4test_lambda_expressionsrc Csztjdkr tdGdddt}tttddddd \}}}Wdn1s.wY|d d ks;J|d ksAJ|tksHJtt/t|ddd d d \}}}|d d d d kseJ|d kskJ|td d kstJWdn1s~wYtt,t t t|ddWdn1swYWddSWddS1swYdS)Nrz8Warning check unstable under Python 2, life is too shortc@seZdZddZddZdS)zAtest_delayed_check_pickle_deprecated..UnpicklableCallablec_dSNrWr1rNrrr1r1r4__call__rTzJtest_delayed_check_pickle_deprecated..UnpicklableCallable.__call__cSstrBrrr1r1r4rszLtest_delayed_check_pickle_deprecated..UnpicklableCallable.__reduce__N)rVrWrXrrr1r1r1r4UnpicklableCallables rcSrrr1rJr1r1r4r sz6test_delayed_check_pickle_deprecated..FrrOrWrrP)optionT) rrrrrrDeprecationWarningr$dictrr[)rrbrrr1r1r4$test_delayed_check_pickle_deprecateds4         "r csd}d}d|td||d}|fddt|D|jj|jjks&J|jj|jjks0J|fddt|D|jj|jjksGJ|jj|jjksQJd S) zCTest that a parallel backend correctly resets its batch statistics.rIig@r)rzrhr@c3|] }ttjVqdSrBrrk task_timer1r4rm#rz6test_backend_batch_statistics_reset..c3r rBrrkrr1r4rm)rN)r#rqr_effective_batch_size_DEFAULT_EFFECTIVE_BATCH_SIZE_smoothed_batch_duration _DEFAULT_SMOOTHED_BATCH_DURATION)r@rhn_inputsrr1rr4#test_backend_batch_statistics_resets$ rcCsdD];}tt|djtksJt|dd}t|jtksJt|dd}t|jtks.Jt|dd}t|jtks=Jqtdd dd }t|jtksNJtd ddtdd }t|jtkscJ|jdksjJWdn1stwYtd ddtd dd}t|jtksJ|jd ksJWdn1swYtd ddtdd }t|jtksJ|jdksJWdn1swYtd ddtd dd}t|jtksJ|jd ksJWddS1swYdS)N)rrIrxrthreadsrhprefer processes sharedmemrhrequirerIr8)rhr@rrrorr)rdr#rr!rr&rh)rhrr1r1r4$test_backend_hinting_and_constraints0s<       "rcCszGdddt}t|&tddd}t|j|ksJtddd}t|j|ks,JWdn1s6wYGdd d t}t|Otddd}t|j|ksXJ|\}}|d ksdJ|d ksjJtddd d }t|jtkszJ|\}}d }||ksJ|d ksJWdn1swYtt t|ddWddS1swYdS)Nc@$eZdZdZdZddZddZdS)z[test_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackendTcSrMrBr1rr1r1r4 apply_asyncerTzgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_asynccS|SrBr1rAr1r1r4r'hrTzltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsNrVrWrXsupports_sharedmem use_threadsr!r'r1r1r1r4MyCustomThreadingBackenda  r&rIrrrrc@r )z\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcSrMrBr1rr1r1r4r!vrTzhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asynccSr"rBr1rAr1r1r4r'yrTzmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsNr#r1r1r1r4MyCustomProcessingBackendrr'r(r)r)rhrrzzUsing ThreadingBackend as joblib.Parallel backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r@r) r r&r#rdr readouterrrrrr[)capsysr&rr(outerrrr1r1r49test_backend_hinting_and_constraints_with_custom_backends^s0        "r.cCstt tddWdn1swYtt tddWdn1s-wYtttdddWdn1sGwYtttdddWdn1sawYtttd ddWddS1s|wYdS) Ninvalidrrrr)rrr8r)r7rBr1r1r1r4,test_invalid_backend_hinting_and_constraintss       "r0cCsFtj}td}ttjtsJ|ttjt|us!JdS)Nr9)r#rr&rr unregisterrd)defaultpbr1r1r4test_global_parallel_backends r4cCsPdd}|td<tdttjtsJWddS1s!wYdS)NcSs ttd<dS)Nfoo)rr*r1r1r1r4 register_foo z,test_external_backends..register_foor5)r,r&rr#rr)r6r1r1r4test_external_backendss  "r8c stdd4}t|jj|jjfg}dkr|WdS|fddtdD}||dWdS1s.r)r#rdrrVrsrq)r9rr this_levelrr1r;r4r:s  $r:cCst|dd t}Wdn1swYtdkr#d}d}nd}d}|d}|df|dfd|fd|fg}||ksAJdS)NrIrrrrBackendr)r&r:r(title)r@backend_types_and_levelssecond_level_backend_type max_leveltop_level_backend_typeexpected_types_and_levelsr1r1r4test_nested_parallelism_limits  rDzThis test requires dask)reasonc Cstjddd}tjtdtjd}tdD].}td t|d}Wdn1s+wYt |dks8Jt dd |DsCJqtd t}Wdn1sVwYt |dkscJt d d |DsnJdS) NrI) n_workersthreads_per_workergcArdask)datarcs|] \}}|dkVqdSDaskDistributedBackendNr1r2rrr1r1r4rmz4test_nested_parallelism_with_dask..csrJrKr1rMr1r1r4rmrN) distributedClientrrr8rrqr&r:rr)clientrIrlr?r1r1r4!test_nested_parallelism_with_dasks"    rRcCstddtdDS)z6A horrible function that does recursive parallel callscsr^rB)r$_recursive_parallelrkr1r1r4rmsz&_recursive_parallel..rIrp) nesting_limitr1r1r4rSsrSc Cspt|dd(tt tWdn1swYWddSWddS1s1wYdSr)r&rRecursionErrorrSr?r1r1r4test_thread_bomb_mitigations "rVcCs*i}dD] }tj|||<q|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSVECLIB_MAXIMUM_THREADSNUMEXPR_NUM_THREADSNUMBA_NUM_THREADS ENABLE_IPCr|)r1r9r:r)env_varsvarr1r1r4_run_parallel_sums r`zNeed OpenMP helper compiledcCstd|dddtdD}ttdd}|D]-\}}||ks#J|D]\}}|dr9|t|ks8Jq'|dks?J|dksEJq'qdS) NrIrgcsr^rB)r$r`rr1r1r4rm  z-test_parallel_thread_limit..r_THREADSr]1)r#rqr@r(itemsendswithr)r@rexpected_num_threadsworker_env_varsomp_num_threadsrrr1r1r4test_parallel_thread_limit s    riz%This test requires dask NOT installedcCs:ttdd tdWddS1swYdS)NzPlease install daskrrH)rr[r&r1r1r1r4)test_dask_backend_when_dask_not_installeds "rjc CsGdddt}d}t|6tjt|dtddddtdDWdn1s0wYWddSWddS1sHwYdS) Nc@s&eZdZddZdddZddZdS) z3test_zero_worker_backend..ZeroWorkerBackendc_rrcr1rr1r1r4r?'rTz=test_zero_worker_backend..ZeroWorkerBackend.configureNcSr)NzNo worker availabler)rNfunccallbackr1r1r4r!*rFz?test_zero_worker_backend..ZeroWorkerBackend.apply_asynccSrrcr1rAr1r1r4r'-rTzDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobsrB)rVrWrXr?r!r'r1r1r1r4ZeroWorkerBackend&s  rmz&ZeroWorkerBackend has no active workerrrIrcsrirBrrkr1r1r4rm3rz+test_zero_worker_backend..)rr&rrrr#rq)rm expected_msgr1r1r4test_zero_worker_backend#s "rocsdadddks JtddfddtdD}t|dhks&Jdadks/Jtddfd dtdD}t|dhksHJdS) Nzoriginal valuecSstSrB)MY_GLOBAL_VARIABLEr1r1r1r4 check_globals>sz@test_globals_update_at_each_parallel_call..check_globalsrIrc3|]}tVqdSrBrorkrqr1r4rmDraz.z changed valuec3rrrBrorkrsr1r4rmMra)rpr#rqrh)workers_global_variabler1rsr4)test_globals_update_at_each_parallel_call6srucCs4ddl}|jdd}|||ddlm}|S)Nrr|)threadpool_info)numpyrrr threadpoolctlrv)rrOrvr1r1r4_check_numpy_threadpool_limitsWs   rycCs4|D]}|d|dkr|dSqtd|)Nfilepath num_threadsz,An unexpected module was loaded in child: {})r[r) child_module parent_info parent_moduler1r1r4_parent_max_num_threads_forbs rcCs@|D]}|D]}t||}t|||h}|d|vsJqqdS)Nr{)rmin) workers_infor}r{child_threadpool_infor|parent_max_num_threadsrr1r1r4check_child_num_threadsjsr)rorz%threadpoolctl is a python3.5+ package)rIrryrxcCsbt}t|dkrtjddt|dddtdD}t|}tt|d}t |||dS) Nr&Need a version of numpy linked to BLASmsgrcsr^rBr$ryrkr1r1r4rmraz6test_threadpool_limitation_in_child..rIr) ryrrrr#rqr'r@r(r)rhr}workers_threadpool_infosexpected_child_num_threadsr1r1r4#test_threadpool_limitation_in_childvs  rinner_max_num_threads)rrIrNrxcCst}t|dkrtjddtd|dt|dddtd D}Wdn1s.wYt|}|durDtt |d }n|}t |||dS) Nrrrr8rrcsr^rBrrkr1r1r4rmraz>test_threadpool_limitation_in_child_context..rIr) ryrrrr&r#rqr'r@r(r)rhrr}rrr1r1r4+test_threadpool_limitation_in_child_contexts   rvar_name)rXrYrWc stddddtj}z_dtj<t|dfddtd D}|ddgks0Jtd d d t|dfd dtd D}Wdn1sRwY|ddgks_JW|durjtj=dS|tj<dS|durztj=w|tj<w)NTreusecSs tj|SrB)r1r9r:)rr1r1r4_get_envr7z>test_threadpool_limitation_in_child_override.._get_env4rc3|] }tVqdSrBrorkrrr1r4rmrnz?test_threadpool_limitation_in_child_override..rIr8rrc3rrBrorkrr1r4rmrnrc)r shutdownr1r9r:r#rqr&)rhroriginal_var_valuerr1rr4,test_threadpool_limitation_in_child_overrides*     rcCs>ttddt|ddWddS1swYdS)Nz#does not acc.*inner_max_num_threadsrrr)rAssertionErrorr&r?r1r1r41test_threadpool_limitation_in_child_context_errors"r)rIrrxcCsJdd}||tdd}tdD]}||tdd}||ks"JqdS)Ncs,tdt|dfddtdDdS)Nrrc3rrB)r$sumrkrJr1r4rmrzAtest_loky_reuse_workers..parallel_call..rrr1rJr4 parallel_calls$z.test_loky_reuse_workers..parallel_callTrr)r rq)rhrfirst_executorrexecutorr1r1r4test_loky_reuse_workerss   r)F)rr)rorB)rYr1rr^rr9 tracebackrmathrrrrr7rrrr r r joblib.externals.lokyr joblib.test.commonrrrjoblib.testingrrrrrrjoblib._compatrrcPickler;rrposixrU NameErrorrr importlib _openmp_test_helper.parallel_sumrrOjoblib._parallel_backendsrrrr r!r"joblib.parallelr#r$r%r&r'r(r)r*r+r,joblib.my_exceptionsr-r.r/r0sortedkeysALL_VALID_BACKENDSPROCESS_BACKENDSPARALLEL_BACKENDShasattrrr:rIrArErKrrLr]r`rbrfrsrurwrrrrrrrrrrrrrrrrrrrrr r"r&r(r.r6r<r=rCrErGrJrPrRextendrqrSrTrYr[r]r`_parallel_backendsrarkrrrvrzrrrrrrrrrrr SQUARE_LOCAL SQUARE_LAMBDArrrrrrrrrrrrrrrrrrrr rrr.r0r4r8r:rDrRrSrVr`rirjroruryrrrrrrrr1r1r1r4s^                                                 I        %                   " $ (            .4          !