o Fh9@sdZddlZddlZddlZddlmZmZddlmZddl m Z ddl m Z ddl mZdZd d ZGd d d ZGd ddeZGdddeZdS)u:class:`.RateLimiter` and :class:`.AsyncRateLimiter` allow to perform bulk operations while gracefully handling error responses and adding delays when needed. In the example below a delay of 1 second (``min_delay_seconds=1``) will be added between each pair of ``geolocator.geocode`` calls; all :class:`geopy.exc.GeocoderServiceError` exceptions will be retried (up to ``max_retries`` times):: import pandas as pd df = pd.DataFrame({'name': ['paris', 'berlin', 'london']}) from geopy.geocoders import Nominatim geolocator = Nominatim(user_agent="specify_your_app_name_here") from geopy.extra.rate_limiter import RateLimiter geocode = RateLimiter(geolocator.geocode, min_delay_seconds=1) df['location'] = df['name'].apply(geocode) df['point'] = df['location'].apply(lambda loc: tuple(loc.point) if loc else None) This would produce the following DataFrame:: >>> df name location \ 0 paris (Paris, Île-de-France, France métropolitaine, ... 1 berlin (Berlin, 10117, Deutschland, (52.5170365, 13.3... 2 london (London, Greater London, England, SW1A 2DU, UK... point 0 (48.8566101, 2.3514992, 0.0) 1 (52.5170365, 13.3888599, 0.0) 2 (51.5073219, -0.1276474, 0.0) To pass extra options to the `geocode` call:: from functools import partial df['location'] = df['name'].apply(partial(geocode, language='de')) To see a progress bar:: from tqdm import tqdm tqdm.pandas() df['location'] = df['name'].progress_apply(geocode) Before using rate limiting classes, please consult with the Geocoding service ToS, which might explicitly consider bulk requests (even throttled) a violation. N)chaincount)sleep default_timer)GeocoderServiceError)logger)AsyncRateLimiter RateLimitercCstddt|DdgS)z-list(_is_last_gen(2)) -> [False, False, True]css|]}dVqdS)FN).0_r r Q/var/www/html/loop/nvenv/lib/python3.10/site-packages/geopy/extra/rate_limiter.py Bsz_is_last_gen..T)rrange)rr r r _is_last_gen@src@s>eZdZdZefZddZddZddZdd Z d d Z d S) BaseRateLimiterz9Base Rate Limiter class for both sync and async versions.cCs8||_||_||_||_|dksJt|_d|_dS)Nr)min_delay_seconds max_retriesswallow_exceptionsreturn_value_on_exception threadingLock_lock _last_call)selfrrrrr r r__init__Js   zBaseRateLimiter.__init__cCstSNr)rr r r_clock\szBaseRateLimiter._clockccs |j7|}|jdur||_ WddS||j}|j|}|dkr5||_ WddSWdn1s?wY|Vq)NTr)rrrr)rclockseconds_since_last_callwaitr r r_acquire_request_slot_gen_s$   z)BaseRateLimiter._acquire_request_slot_genc cs|ttt|jD]1\}}z|VWdS|jy;|r#dVntjt|jd||j||dddVYq Yq wdS)NTzB caught an error, retrying (%s/%s tries). Called with (*%r, **%r).exc_infoF) ziprrr_retry_exceptionsrwarningtype__name__)rargskwargsi is_last_tryr r r _retries_gens*  zBaseRateLimiter._retries_gencCs.|jrtjt|jd|j||dd|jS)Nz> swallowed an error after %r retries. Called with (*%r, **%r).Tr#)rrr'r(r)rr)rr*r+r r r _handle_excs zBaseRateLimiter._handle_excN) r) __module__ __qualname____doc__rr&rrr"r.r/r r r rrEs& rcFeZdZdZddddddfdd Zd d Zd d ZddZZS)r aThis is a Rate Limiter implementation for synchronous functions (like geocoders with the default :class:`geopy.adapters.BaseSyncAdapter`). Examples:: from geopy.extra.rate_limiter import RateLimiter from geopy.geocoders import Nominatim geolocator = Nominatim(user_agent="specify_your_app_name_here") search = ["moscow", "paris", "berlin", "tokyo", "beijing"] geocode = RateLimiter(geolocator.geocode, min_delay_seconds=1) locations = [geocode(s) for s in search] search = [ (55.47, 37.32), (48.85, 2.35), (52.51, 13.38), (34.69, 139.40), (39.90, 116.39) ] reverse = RateLimiter(geolocator.reverse, min_delay_seconds=1) locations = [reverse(s) for s in search] RateLimiter class is thread-safe. If geocoding service's responses are slower than `min_delay_seconds`, then you can benefit from parallelizing the work:: import concurrent.futures geolocator = OpenMapQuest(api_key="...") geocode = RateLimiter(geolocator.geocode, min_delay_seconds=1/20) with concurrent.futures.ThreadPoolExecutor() as e: locations = list(e.map(geocode, search)) .. versionchanged:: 2.0 Added thread-safety support. @TNrrerror_wait_secondsrrc<tj||||d||_||_||ksJ|dksJdSa  :param callable func: A function which should be wrapped by the rate limiter. :param float min_delay_seconds: Minimum delay in seconds between the wrapped ``func`` calls. To convert :abbr:`RPS (Requests Per Second)` rate to ``min_delay_seconds`` you need to divide 1 by RPS. For example, if you need to keep the rate at 20 RPS, you can use ``min_delay_seconds=1/20``. :param int max_retries: Number of retries on exceptions. Only :class:`geopy.exc.GeocoderServiceError` exceptions are retried -- others are always re-raised. ``max_retries + 1`` requests would be performed at max per query. Set ``max_retries=0`` to disable retries. :param float error_wait_seconds: Time to wait between retries after errors. Must be greater or equal to ``min_delay_seconds``. :param bool swallow_exceptions: Should an exception be swallowed after retries? If not, it will be re-raised. If yes, the ``return_value_on_exception`` will be returned. :param return_value_on_exception: Value to return on failure when ``swallow_exceptions=True``. )rrrrrNsuperrfuncr8rr=rrr8rr __class__r rr) zRateLimiter.__init__cCs"tt|jd|t|dSNz sleep(%r))rdebugr(r)rrsecondsr r r_sleeps zRateLimiter._sleepcCs|D]}||qdSrr"rFrr!r r r_acquire_request_slot s  z!RateLimiter._acquire_request_slotc Os|||}|D]H}|z|j|i|}t|r td|WS|jyP}z||r@|||WYd}~S| |j WYd}~qd}~wwt d)NzoAn async awaitable has been passed to `RateLimiter`. Use `AsyncRateLimiter` instead, which supports awaitables.Should not have been reached) r.rIr=inspect isawaitable ValueErrorr&throwr/rFr8 RuntimeError)rr*r+genr reser r r__call__ s"    zRateLimiter.__call__ r)r0r1r2rrFrIrS __classcell__r r r?rr s)4r cr3)r aThis is a Rate Limiter implementation for asynchronous functions (like geocoders with :class:`geopy.adapters.BaseAsyncAdapter`). Examples:: from geopy.adapters import AioHTTPAdapter from geopy.extra.rate_limiter import AsyncRateLimiter from geopy.geocoders import Nominatim async with Nominatim( user_agent="specify_your_app_name_here", adapter_factory=AioHTTPAdapter, ) as geolocator: search = ["moscow", "paris", "berlin", "tokyo", "beijing"] geocode = AsyncRateLimiter(geolocator.geocode, min_delay_seconds=1) locations = [await geocode(s) for s in search] search = [ (55.47, 37.32), (48.85, 2.35), (52.51, 13.38), (34.69, 139.40), (39.90, 116.39) ] reverse = AsyncRateLimiter(geolocator.reverse, min_delay_seconds=1) locations = [await reverse(s) for s in search] AsyncRateLimiter class is safe to use across multiple concurrent tasks. If geocoding service's responses are slower than `min_delay_seconds`, then you can benefit from parallelizing the work:: import asyncio async with OpenMapQuest( api_key="...", adapter_factory=AioHTTPAdapter ) as geolocator: geocode = AsyncRateLimiter(geolocator.geocode, min_delay_seconds=1/20) locations = await asyncio.gather(*(geocode(s) for s in search)) .. versionadded:: 2.0 r4r5r6TNr7cr9r:r;r>r?r rrLrAzAsyncRateLimiter.__init__cs,tt|jd|t|IdHdSrB)rrCr(r)asynciorrDr r rrFszAsyncRateLimiter._sleepcs$|D] }||IdHqdSrrGrHr r rrIs z&AsyncRateLimiter._acquire_request_slotc s|||}|D]F}|IdHz|j|i|IdHWS|jyO}z"||r<|||WYd}~S||jIdHWYd}~q d}~wwtd)NrJ) r.rIr=r&rNr/rFr8rO)rr*r+rPr rRr r rrSs  zAsyncRateLimiter.__call__rTr r r?rr "s-4r )r2rVrKr itertoolsrrtimertimeitr geopy.excr geopy.utilr__all__rrr r r r r rs2    fw