|
|
@ -1,6 +1,7 @@ |
|
|
|
import aiofiles |
|
|
|
import aiohttp |
|
|
|
import asyncio |
|
|
|
import backoff |
|
|
|
import click |
|
|
|
import click_log |
|
|
|
import hashlib |
|
|
@ -165,6 +166,7 @@ async def check_integrity(path, expected_checksum): |
|
|
|
return expected_checksum == hash.hexdigest() |
|
|
|
|
|
|
|
|
|
|
|
@backoff.on_exception(backoff.expo, aiohttp.ServerDisconnectedError, max_time=60) |
|
|
|
async def get_files_data(identifier, session): |
|
|
|
url = f"https://archive.org/metadata/{identifier}/files" |
|
|
|
logger.debug(f"Fetching files data at {url}...") |
|
|
@ -172,6 +174,7 @@ async def get_files_data(identifier, session): |
|
|
|
return await response.json() |
|
|
|
|
|
|
|
|
|
|
|
@backoff.on_exception(backoff.expo, aiohttp.ServerDisconnectedError, max_time=60) |
|
|
|
async def download_file(path, url, session, semaphore): |
|
|
|
async with semaphore: |
|
|
|
logger.debug(f"Downloading file {url}...") |
|
|
|