Browse Source

Added retry logic

master
Eliot Berriot 2 years ago
parent
commit
8aea944221
No known key found for this signature in database GPG Key ID: DD6965E2476E5C27
2 changed files with 4 additions and 0 deletions
  1. +3
    -0
      archiveorg_dl/__init__.py
  2. +1
    -0
      setup.cfg

+ 3
- 0
archiveorg_dl/__init__.py View File

@ -1,6 +1,7 @@
import aiofiles
import aiohttp
import asyncio
import backoff
import click
import click_log
import hashlib
@ -165,6 +166,7 @@ async def check_integrity(path, expected_checksum):
return expected_checksum == hash.hexdigest()
@backoff.on_exception(backoff.expo, aiohttp.ServerDisconnectedError, max_time=60)
async def get_files_data(identifier, session):
url = f"https://archive.org/metadata/{identifier}/files"
logger.debug(f"Fetching files data at {url}...")
@ -172,6 +174,7 @@ async def get_files_data(identifier, session):
return await response.json()
@backoff.on_exception(backoff.expo, aiohttp.ServerDisconnectedError, max_time=60)
async def download_file(path, url, session, semaphore):
async with semaphore:
logger.debug(f"Downloading file {url}...")


+ 1
- 0
setup.cfg View File

@ -23,6 +23,7 @@ install_requires =
click-log
aiofiles
aiohttp
backoff
[options.entry_points]
console_scripts =


Loading…
Cancel
Save