Eliot Berriot
3 years ago
No known key found for this signature in database
GPG Key ID: DD6965E2476E5C27
2 changed files with
4 additions and
0 deletions
-
archiveorg_dl/__init__.py
-
setup.cfg
|
|
|
@ -1,6 +1,7 @@ |
|
|
|
import aiofiles |
|
|
|
import aiohttp |
|
|
|
import asyncio |
|
|
|
import backoff |
|
|
|
import click |
|
|
|
import click_log |
|
|
|
import hashlib |
|
|
|
@ -165,6 +166,7 @@ async def check_integrity(path, expected_checksum): |
|
|
|
return expected_checksum == hash.hexdigest() |
|
|
|
|
|
|
|
|
|
|
|
@backoff.on_exception(backoff.expo, aiohttp.ServerDisconnectedError, max_time=60) |
|
|
|
async def get_files_data(identifier, session): |
|
|
|
url = f"https://archive.org/metadata/{identifier}/files" |
|
|
|
logger.debug(f"Fetching files data at {url}...") |
|
|
|
@ -172,6 +174,7 @@ async def get_files_data(identifier, session): |
|
|
|
return await response.json() |
|
|
|
|
|
|
|
|
|
|
|
@backoff.on_exception(backoff.expo, aiohttp.ServerDisconnectedError, max_time=60) |
|
|
|
async def download_file(path, url, session, semaphore): |
|
|
|
async with semaphore: |
|
|
|
logger.debug(f"Downloading file {url}...") |
|
|
|
|
|
|
|
@ -23,6 +23,7 @@ install_requires = |
|
|
|
click-log |
|
|
|
aiofiles |
|
|
|
aiohttp |
|
|
|
backoff |
|
|
|
|
|
|
|
[options.entry_points] |
|
|
|
console_scripts = |
|
|
|
|
