Skip to content

fix: download dataset atomically so an interrupted download can't block retries #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
steps-re wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: download dataset atomically so an interrupted download can't block retries #8

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 39 additions & 21 deletions download_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import hashlib
import os
import sys

import requests
from torch.utils.model_zoo import tqdm
Expand Down Expand Up @@ -33,27 +34,44 @@ def sha256(target: str) -> str:

target = os.path.join(os.path.dirname(os.path.abspath(__file__)), "spectrum_sensing_dataset.hdf5")

# Check if the dataset source file already exists.
# Check if the dataset source file already exists. Only skip the download if the existing file also
# matches the expected checksum; otherwise it is an incomplete or corrupted download and we re-fetch it.
if os.path.exists(target):
print(f"{target} already exists. Aborting download.")
exit(1)

# Download the dataset source file into the target directory.
print(f"Downloading {format(file_url)}")
n_bytes = int(requests.head(file_url).headers.get("Content-Length", 0))

with (
requests.get(file_url, stream=True, timeout=3) as r,
open(target, "wb") as out_file,
tqdm(desc="Downloading MathWorks' Spectrum Sensing Dataset", total=n_bytes, unit="B", unit_scale=True) as pbar,
):
for chunk in r.iter_content(chunk_size=1024):
if chunk:
out_file.write(chunk)
pbar.update(len(chunk))

if sha256_checksum != sha256(target=target):
if sha256(target=target) == sha256_checksum:
print(f"{target} already exists and matches the expected checksum. Nothing to do.")
sys.exit(0)
print(f"{target} already exists but does not match the expected checksum. Re-downloading.")

# Download into a temporary '.part' file first, so that an interrupted or failed download never leaves a
# corrupted file at the target path (which would otherwise block all future download attempts).
partial = target + ".part"

print(f"Downloading {file_url}")
n_bytes = int(requests.head(file_url, timeout=30).headers.get("Content-Length", 0))

try:
with (
requests.get(file_url, stream=True, timeout=30) as r,
open(partial, "wb") as out_file,
tqdm(desc="Downloading MathWorks' Spectrum Sensing Dataset", total=n_bytes, unit="B", unit_scale=True) as pbar,
):
r.raise_for_status()
for chunk in r.iter_content(chunk_size=1024):
if chunk:
out_file.write(chunk)
pbar.update(len(chunk))
except BaseException:
# Remove the partial file on any error or interruption (e.g. Ctrl+C) so a retry starts clean.
if os.path.exists(partial):
os.remove(partial)
raise

if sha256(target=partial) != sha256_checksum:
os.remove(partial)
raise RuntimeError(
f"Checksum of {target} does not match expected.\n"
f"The download may be corrupted, please remove the corrupted resource and try again."
"Checksum of the downloaded file does not match expected.\n"
"The download may be corrupted, please try again."
)

# Atomically move the fully-downloaded, checksum-verified file into place.
os.replace(partial, target)