Ensemble Forecasting with Downscaling¶
Custom ensembling workflow with generative downscaling using CorrDiff.
This example demonstrates an ensemble forecasting pipeline that runs a prognostic model ensemble and applies CorrDiff generative downscaling to each step and member. While this example uses SFNO for the prognostic model, the pipeline is model-agnostic and can be reused with other prognostic models that follow the Earth2Studio interfaces.
In this example you will learn:
- How to create an ensemble forecast pipeline with CorrDiff downscaling
- Saving output ensemble data to a Zarr store
- Post-process results
Creating an Ensemble Downscaling Workflow¶
To create our own ensemble forecasting with downscaling workflow, we will use the
built-in ensemble workflow earth2studio.run.ensemble as the reference to
start with. For this to work we need to update how the output coordinates are
calculated for the IO object, as well as add the CorrDiff model's forward call
into the forecast loop.
As in previous examples, we use dependency injection to define the signature of the pipeline method.
import os
os.makedirs("outputs", exist_ok=True)
from dotenv import load_dotenv
load_dotenv() # TODO: make common example prep function
from datetime import datetime
from math import ceil
import numpy as np
import torch
from loguru import logger
from tqdm import tqdm
from earth2studio.data import DataSource, fetch_data
from earth2studio.io import IOBackend
from earth2studio.models.dx import CorrDiff
from earth2studio.models.px import PrognosticModel
from earth2studio.perturbation import Perturbation
from earth2studio.utils.coords import map_coords, split_coords
from earth2studio.utils.time import to_time_array
def corrdiff_on_hens_ensemble(
time: list[str] | list[datetime] | list[np.datetime64],
nsteps: int,
nensemble: int,
nsamples: int,
prognostic: PrognosticModel,
corrdiff: CorrDiff,
data: DataSource,
io: IOBackend,
perturbation: Perturbation,
batch_size: int | None = None,
) -> IOBackend:
"""Ensemble CorrDiff pipeline
Parameters
----------
time : list[str] | list[datetime] | list[np.datetime64]
Forecast start times
nsteps : int
Number of forecast steps for prognostic model to take
nensemble : int
Number of forecast ensemble members
nsamples : int
Number of samples from CorrDiff model to generate
prognostic : PrognosticModel
Prognostic model
corrdiff : CorrDiff
CorrDiff model
data : DataSource
Data source
io : IOBackend
IO Backend
perturbation : Perturbation
Perturbation method
batch_size : int | None, optional
Ensemble batch size during forecasting. If None, uses nensemble; default None
"""
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
prognostic = prognostic.to(device)
corrdiff = corrdiff.to(device)
corrdiff.number_of_samples = nsamples
# Fetch initial data for the ensemble
prognostic_ic = prognostic.input_coords()
time = to_time_array(time)
x0, coords0 = fetch_data(
source=data,
time=time,
variable=prognostic_ic["variable"],
lead_time=prognostic_ic["lead_time"],
device=device,
interp_to=prognostic_ic if hasattr(prognostic, "interp_method") else None,
interp_method=getattr(prognostic, "interp_method", "nearest"),
)
# Prepare CorrDiff output coordinates for IO backend
total_coords = corrdiff.output_coords(corrdiff.input_coords())
if "batch" in total_coords:
del total_coords["batch"]
total_coords["time"] = time
total_coords["lead_time"] = np.asarray(
[
prognostic.output_coords(prognostic.input_coords())["lead_time"] * i
for i in range(nsteps + 1)
]
).flatten()
total_coords["ensemble"] = np.arange(nensemble)
total_coords.move_to_end("lead_time", last=False)
total_coords.move_to_end("time", last=False)
total_coords.move_to_end("ensemble", last=False)
variables_to_save = total_coords.pop("variable")
io.add_array(total_coords, variables_to_save)
# Determine batch size and number of batches
batch_size = min(nensemble, batch_size or nensemble)
number_of_batches = ceil(nensemble / batch_size)
logger.info(
f"Starting {nensemble} member ensemble inference with {number_of_batches} batches."
)
# Main ensemble loop
for batch_id in tqdm(
range(0, nensemble, batch_size),
total=number_of_batches,
desc="Ensemble Batches",
):
mini_batch_size = min(batch_size, nensemble - batch_id)
x = x0.to(device)
# Set up coordinates for this batch
coords = {
"ensemble": np.arange(batch_id, batch_id + mini_batch_size),
**coords0.copy(),
}
# Repeat initial condition for each ensemble member in the batch
x = x.unsqueeze(0).repeat(mini_batch_size, *([1] * x.ndim))
x, coords = map_coords(x, coords, prognostic_ic)
x, coords = perturbation(x, coords)
model = prognostic.create_iterator(x, coords)
with tqdm(
total=nsteps + 1,
desc=f"Batch {batch_id} inference",
position=1,
leave=False,
) as pbar:
for step, (x, coords) in enumerate(model):
# Map prognostic outputs to CorrDiff inputs if needed
x, coords = map_coords(x, coords, corrdiff.input_coords())
# CorrDiff workflow: generate and write CorrDiff outputs
x, coords = corrdiff(x, coords)
io.write(*split_coords(x, coords))
pbar.update(1)
if step == nsteps:
break
logger.success("Inference complete")
return io
Console output9 lines
/__w/earth2studio/earth2studio/.venv/lib/python3.13/site-packages/torch/cuda/__init__.py:64: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
import pynvml # type: ignore[import]
WARNING[XFORMERS]: xFormers can't load C++/CUDA extensions. xFormers was built for:
PyTorch 2.10.0+cu128 with CUDA 1208 (you have 2.13.0+cu130)
Python 3.10.19 (you have 3.13.13)
Please reinstall xformers (see https://github.com/facebookresearch/xformers#installing-xformers)
Memory-efficient attention, SwiGLU, sparse and more won't be available.
Set XFORMERS_MORE_DETAILS=1 for more details
CuPy distance computation test failed with error: cuVS >= 24.12 or pylibraft < 24.12 should be installed to use this featureSet Up¶
With the inference pipeline function defined, next let's create the required components as usual. We need the following:
- Prognostic Model: Use the built in SFNO model
earth2studio.models.px.SFNO. - CorrDiff Model: Use the built in CorrDiff Taiwan model
earth2studio.models.dx.CorrDiffTaiwan. - Datasource: Pull data from the GFS data api
earth2studio.data.GFS. - IO Backend: Let's save the outputs into a Zarr store
earth2studio.io.ZarrBackend.
For the prognostic checkpoint, we will use a HENS checkpoint conveniently stored on HuggingFace. Refer to the previous examples for more information about loading these models.
from earth2studio.data import GFS
from earth2studio.io import ZarrBackend
from earth2studio.models.auto import Package
from earth2studio.models.dx import CorrDiffTaiwan
from earth2studio.models.px import SFNO
from earth2studio.perturbation import (
CorrelatedSphericalGaussian,
HemisphericCentredBredVector,
)
# Create data source
data = GFS()
# Load prognostic model
hens_package = Package(
"hf://datasets/maheshankur10/hens/earth2mip_prod_registry/sfno_linear_74chq_sc2_layers8_edim620_wstgl2-epoch70_seed103",
cache_options={
"cache_storage": Package.default_cache("hens_1"),
"same_names": True,
},
)
model = SFNO.load_model(hens_package)
# Set up perturbation method
noise_amplification = torch.zeros(model.input_coords()["variable"].shape[0])
index_z500 = list(model.input_coords()["variable"]).index("z500")
noise_amplification[index_z500] = 39.27
noise_amplification = noise_amplification.reshape(1, 1, 1, -1, 1, 1)
seed_perturbation = CorrelatedSphericalGaussian(noise_amplitude=noise_amplification)
perturbation = HemisphericCentredBredVector(
model, data, seed_perturbation, noise_amplitude=noise_amplification
)
# Load the CorrDiffTaiwan model
corrdiff = CorrDiffTaiwan.load_model(CorrDiffTaiwan.load_default_package())
# Set up IO backend
io = ZarrBackend(
file_name="outputs/18_ensemble_corrdiff.zarr",
chunks={"ensemble": 1, "sample": 1, "time": 1, "lead_time": 1},
backend_kwargs={"overwrite": True},
)
Console output873 lines
Downloading config.json: 0%| | 0.00/22.4k [00:00<?, ?B/s]
Downloading config.json: 100%|██████████| 22.4k/22.4k [00:00<00:00, 219kB/s]
Downloading config.json: 100%|██████████| 22.4k/22.4k [00:00<00:00, 216kB/s]
Downloading global_means.npy: 0%| | 0.00/720 [00:00<?, ?B/s]
Downloading global_means.npy: 100%|██████████| 720/720 [00:00<00:00, 1.55kB/s]
Downloading global_means.npy: 100%|██████████| 720/720 [00:00<00:00, 1.54kB/s]
Downloading global_stds.npy: 0%| | 0.00/720 [00:00<?, ?B/s]
Downloading global_stds.npy: 100%|██████████| 720/720 [00:00<00:00, 1.38kB/s]
Downloading global_stds.npy: 100%|██████████| 720/720 [00:00<00:00, 1.38kB/s]
Downloading orography.nc: 0%| | 0.00/2.50M [00:00<?, ?B/s]
Downloading orography.nc: 100%|██████████| 2.50M/2.50M [00:00<00:00, 5.37MB/s]
Downloading orography.nc: 100%|██████████| 2.50M/2.50M [00:00<00:00, 5.32MB/s]
Downloading land_mask.nc: 0%| | 0.00/748k [00:00<?, ?B/s]
Downloading land_mask.nc: 100%|██████████| 748k/748k [00:00<00:00, 2.39MB/s]
Downloading land_mask.nc: 100%|██████████| 748k/748k [00:00<00:00, 2.37MB/s]
Downloading best_ckpt_mp0.tar: 0%| | 0.00/8.31G [00:00<?, ?B/s]
Downloading best_ckpt_mp0.tar: 0%| | 10.0M/8.31G [00:01<16:40, 8.90MB/s]
Downloading best_ckpt_mp0.tar: 0%| | 20.0M/8.31G [00:01<08:25, 17.6MB/s]
Downloading best_ckpt_mp0.tar: 0%| | 30.0M/8.31G [00:01<05:39, 26.2MB/s]
Downloading best_ckpt_mp0.tar: 0%| | 40.0M/8.31G [00:01<04:48, 30.8MB/s]
Downloading best_ckpt_mp0.tar: 1%| | 50.0M/8.31G [00:01<03:56, 37.5MB/s]
Downloading best_ckpt_mp0.tar: 1%| | 60.0M/8.31G [00:02<04:00, 36.8MB/s]
Downloading best_ckpt_mp0.tar: 1%| | 70.0M/8.31G [00:02<04:16, 34.5MB/s]
Downloading best_ckpt_mp0.tar: 1%| | 80.0M/8.31G [00:02<03:44, 39.4MB/s]
Downloading best_ckpt_mp0.tar: 1%| | 90.0M/8.31G [00:02<03:20, 43.9MB/s]
Downloading best_ckpt_mp0.tar: 1%| | 100M/8.31G [00:03<03:12, 45.8MB/s]
Downloading best_ckpt_mp0.tar: 1%|▏ | 110M/8.31G [00:03<03:03, 48.0MB/s]
Downloading best_ckpt_mp0.tar: 1%|▏ | 120M/8.31G [00:03<02:46, 52.7MB/s]
Downloading best_ckpt_mp0.tar: 2%|▏ | 130M/8.31G [00:03<03:05, 47.3MB/s]
Downloading best_ckpt_mp0.tar: 2%|▏ | 140M/8.31G [00:03<02:45, 53.2MB/s]
Downloading best_ckpt_mp0.tar: 2%|▏ | 150M/8.31G [00:04<02:52, 50.8MB/s]
Downloading best_ckpt_mp0.tar: 2%|▏ | 160M/8.31G [00:04<02:39, 54.8MB/s]
Downloading best_ckpt_mp0.tar: 2%|▏ | 170M/8.31G [00:04<02:43, 53.5MB/s]
Downloading best_ckpt_mp0.tar: 2%|▏ | 180M/8.31G [00:04<02:30, 58.2MB/s]
Downloading best_ckpt_mp0.tar: 2%|▏ | 190M/8.31G [00:04<02:39, 54.7MB/s]
Downloading best_ckpt_mp0.tar: 2%|▏ | 200M/8.31G [00:05<03:08, 46.2MB/s]
Downloading best_ckpt_mp0.tar: 2%|▏ | 210M/8.31G [00:05<02:51, 50.9MB/s]
Downloading best_ckpt_mp0.tar: 3%|▎ | 220M/8.31G [00:05<02:37, 55.2MB/s]
Downloading best_ckpt_mp0.tar: 3%|▎ | 230M/8.31G [00:05<02:28, 58.4MB/s]
Downloading best_ckpt_mp0.tar: 3%|▎ | 240M/8.31G [00:05<02:20, 61.5MB/s]
Downloading best_ckpt_mp0.tar: 3%|▎ | 250M/8.31G [00:05<02:08, 67.6MB/s]
Downloading best_ckpt_mp0.tar: 3%|▎ | 260M/8.31G [00:06<02:57, 48.7MB/s]
Downloading best_ckpt_mp0.tar: 3%|▎ | 270M/8.31G [00:06<02:41, 53.4MB/s]
Downloading best_ckpt_mp0.tar: 3%|▎ | 280M/8.31G [00:06<02:56, 48.9MB/s]
Downloading best_ckpt_mp0.tar: 3%|▎ | 290M/8.31G [00:06<03:01, 47.4MB/s]
Downloading best_ckpt_mp0.tar: 4%|▎ | 300M/8.31G [00:07<02:57, 48.6MB/s]
Downloading best_ckpt_mp0.tar: 4%|▎ | 310M/8.31G [00:07<03:02, 47.2MB/s]
Downloading best_ckpt_mp0.tar: 4%|▍ | 320M/8.31G [00:07<03:50, 37.2MB/s]
Downloading best_ckpt_mp0.tar: 4%|▍ | 330M/8.31G [00:07<03:30, 40.8MB/s]
Downloading best_ckpt_mp0.tar: 4%|▍ | 340M/8.31G [00:08<03:16, 43.5MB/s]
Downloading best_ckpt_mp0.tar: 4%|▍ | 350M/8.31G [00:08<02:54, 49.1MB/s]
Downloading best_ckpt_mp0.tar: 4%|▍ | 360M/8.31G [00:08<02:44, 52.0MB/s]
Downloading best_ckpt_mp0.tar: 4%|▍ | 370M/8.31G [00:08<02:42, 52.5MB/s]
Downloading best_ckpt_mp0.tar: 4%|▍ | 380M/8.31G [00:09<03:06, 45.6MB/s]
Downloading best_ckpt_mp0.tar: 5%|▍ | 390M/8.31G [00:09<03:32, 40.1MB/s]
Downloading best_ckpt_mp0.tar: 5%|▍ | 400M/8.31G [00:09<03:12, 44.2MB/s]
Downloading best_ckpt_mp0.tar: 5%|▍ | 410M/8.31G [00:09<02:56, 48.0MB/s]
Downloading best_ckpt_mp0.tar: 5%|▍ | 420M/8.31G [00:09<03:01, 46.6MB/s]
Downloading best_ckpt_mp0.tar: 5%|▌ | 430M/8.31G [00:10<02:42, 52.1MB/s]
Downloading best_ckpt_mp0.tar: 5%|▌ | 440M/8.31G [00:10<02:51, 49.3MB/s]
Downloading best_ckpt_mp0.tar: 5%|▌ | 450M/8.31G [00:10<03:20, 42.2MB/s]
Downloading best_ckpt_mp0.tar: 5%|▌ | 460M/8.31G [00:10<03:07, 45.1MB/s]
Downloading best_ckpt_mp0.tar: 6%|▌ | 470M/8.31G [00:11<03:00, 46.7MB/s]
Downloading best_ckpt_mp0.tar: 6%|▌ | 480M/8.31G [00:11<02:51, 49.1MB/s]
Downloading best_ckpt_mp0.tar: 6%|▌ | 490M/8.31G [00:11<02:50, 49.2MB/s]
Downloading best_ckpt_mp0.tar: 6%|▌ | 500M/8.31G [00:11<02:38, 53.1MB/s]
Downloading best_ckpt_mp0.tar: 6%|▌ | 510M/8.31G [00:11<02:36, 53.5MB/s]
Downloading best_ckpt_mp0.tar: 6%|▌ | 520M/8.31G [00:12<02:58, 46.9MB/s]
Downloading best_ckpt_mp0.tar: 6%|▌ | 530M/8.31G [00:12<02:40, 52.2MB/s]
Downloading best_ckpt_mp0.tar: 6%|▋ | 540M/8.31G [00:12<02:32, 54.7MB/s]
Downloading best_ckpt_mp0.tar: 6%|▋ | 550M/8.31G [00:12<02:19, 60.0MB/s]
Downloading best_ckpt_mp0.tar: 7%|▋ | 560M/8.31G [00:12<02:31, 55.0MB/s]
Downloading best_ckpt_mp0.tar: 7%|▋ | 570M/8.31G [00:12<02:32, 54.7MB/s]
Downloading best_ckpt_mp0.tar: 7%|▋ | 580M/8.31G [00:13<02:59, 46.3MB/s]
Downloading best_ckpt_mp0.tar: 7%|▋ | 590M/8.31G [00:13<02:53, 47.9MB/s]
Downloading best_ckpt_mp0.tar: 7%|▋ | 600M/8.31G [00:13<02:45, 50.2MB/s]
Downloading best_ckpt_mp0.tar: 7%|▋ | 610M/8.31G [00:13<02:42, 51.0MB/s]
Downloading best_ckpt_mp0.tar: 7%|▋ | 620M/8.31G [00:14<02:27, 55.9MB/s]
Downloading best_ckpt_mp0.tar: 7%|▋ | 630M/8.31G [00:14<02:24, 57.0MB/s]
Downloading best_ckpt_mp0.tar: 8%|▊ | 640M/8.31G [00:14<02:53, 47.5MB/s]
Downloading best_ckpt_mp0.tar: 8%|▊ | 650M/8.31G [00:14<02:47, 49.3MB/s]
Downloading best_ckpt_mp0.tar: 8%|▊ | 660M/8.31G [00:14<02:38, 51.9MB/s]
Downloading best_ckpt_mp0.tar: 8%|▊ | 670M/8.31G [00:15<02:25, 56.5MB/s]
Downloading best_ckpt_mp0.tar: 8%|▊ | 680M/8.31G [00:15<02:11, 62.2MB/s]
Downloading best_ckpt_mp0.tar: 8%|▊ | 690M/8.31G [00:15<02:08, 63.7MB/s]
Downloading best_ckpt_mp0.tar: 8%|▊ | 700M/8.31G [00:15<02:08, 63.9MB/s]
Downloading best_ckpt_mp0.tar: 8%|▊ | 710M/8.31G [00:15<02:58, 45.8MB/s]
Downloading best_ckpt_mp0.tar: 8%|▊ | 720M/8.31G [00:16<02:56, 46.2MB/s]
Downloading best_ckpt_mp0.tar: 9%|▊ | 730M/8.31G [00:16<02:52, 47.2MB/s]
Downloading best_ckpt_mp0.tar: 9%|▊ | 740M/8.31G [00:16<02:41, 50.5MB/s]
Downloading best_ckpt_mp0.tar: 9%|▉ | 750M/8.31G [00:16<02:29, 54.6MB/s]
Downloading best_ckpt_mp0.tar: 9%|▉ | 760M/8.31G [00:16<02:32, 53.1MB/s]
Downloading best_ckpt_mp0.tar: 9%|▉ | 770M/8.31G [00:17<02:58, 45.5MB/s]
Downloading best_ckpt_mp0.tar: 9%|▉ | 780M/8.31G [00:17<02:36, 51.9MB/s]
Downloading best_ckpt_mp0.tar: 9%|▉ | 790M/8.31G [00:17<02:29, 54.2MB/s]
Downloading best_ckpt_mp0.tar: 9%|▉ | 800M/8.31G [00:17<02:17, 58.8MB/s]
Downloading best_ckpt_mp0.tar: 10%|▉ | 810M/8.31G [00:17<02:24, 55.9MB/s]
Downloading best_ckpt_mp0.tar: 10%|▉ | 820M/8.31G [00:17<02:19, 58.0MB/s]
Downloading best_ckpt_mp0.tar: 10%|▉ | 830M/8.31G [00:18<02:20, 57.3MB/s]
Downloading best_ckpt_mp0.tar: 10%|▉ | 840M/8.31G [00:18<02:44, 48.8MB/s]
Downloading best_ckpt_mp0.tar: 10%|▉ | 850M/8.31G [00:18<02:33, 52.1MB/s]
Downloading best_ckpt_mp0.tar: 10%|█ | 860M/8.31G [00:18<02:23, 55.8MB/s]
Downloading best_ckpt_mp0.tar: 10%|█ | 870M/8.31G [00:18<02:27, 54.1MB/s]
Downloading best_ckpt_mp0.tar: 10%|█ | 880M/8.31G [00:19<02:21, 56.4MB/s]
Downloading best_ckpt_mp0.tar: 10%|█ | 890M/8.31G [00:19<02:10, 61.1MB/s]
Downloading best_ckpt_mp0.tar: 11%|█ | 900M/8.31G [00:19<02:32, 52.2MB/s]
Downloading best_ckpt_mp0.tar: 11%|█ | 910M/8.31G [00:19<02:16, 58.5MB/s]
Downloading best_ckpt_mp0.tar: 11%|█ | 920M/8.31G [00:20<03:00, 44.0MB/s]
Downloading best_ckpt_mp0.tar: 11%|█ | 930M/8.31G [00:20<02:37, 50.3MB/s]
Downloading best_ckpt_mp0.tar: 11%|█ | 940M/8.31G [00:20<02:15, 58.4MB/s]
Downloading best_ckpt_mp0.tar: 11%|█ | 950M/8.31G [00:20<02:00, 65.6MB/s]
Downloading best_ckpt_mp0.tar: 11%|█▏ | 960M/8.31G [00:20<02:42, 48.7MB/s]
Downloading best_ckpt_mp0.tar: 11%|█▏ | 970M/8.31G [00:20<02:44, 47.9MB/s]
Downloading best_ckpt_mp0.tar: 12%|█▏ | 980M/8.31G [00:21<02:30, 52.5MB/s]
Downloading best_ckpt_mp0.tar: 12%|█▏ | 990M/8.31G [00:21<02:28, 53.1MB/s]
Downloading best_ckpt_mp0.tar: 12%|█▏ | 0.98G/8.31G [00:21<02:25, 53.9MB/s]
Downloading best_ckpt_mp0.tar: 12%|█▏ | 0.99G/8.31G [00:21<02:27, 53.3MB/s]
Downloading best_ckpt_mp0.tar: 12%|█▏ | 1.00G/8.31G [00:21<02:25, 54.0MB/s]
Downloading best_ckpt_mp0.tar: 12%|█▏ | 1.01G/8.31G [00:22<03:00, 43.4MB/s]
Downloading best_ckpt_mp0.tar: 12%|█▏ | 1.02G/8.31G [00:22<02:42, 48.3MB/s]
Downloading best_ckpt_mp0.tar: 12%|█▏ | 1.03G/8.31G [00:22<02:37, 49.6MB/s]
Downloading best_ckpt_mp0.tar: 12%|█▏ | 1.04G/8.31G [00:22<02:30, 51.7MB/s]
Downloading best_ckpt_mp0.tar: 13%|█▎ | 1.04G/8.31G [00:22<02:18, 56.1MB/s]
Downloading best_ckpt_mp0.tar: 13%|█▎ | 1.05G/8.31G [00:23<02:20, 55.5MB/s]
Downloading best_ckpt_mp0.tar: 13%|█▎ | 1.06G/8.31G [00:23<02:50, 45.7MB/s]
Downloading best_ckpt_mp0.tar: 13%|█▎ | 1.07G/8.31G [00:23<02:28, 52.2MB/s]
Downloading best_ckpt_mp0.tar: 13%|█▎ | 1.08G/8.31G [00:23<02:22, 54.3MB/s]
Downloading best_ckpt_mp0.tar: 13%|█▎ | 1.09G/8.31G [00:23<02:17, 56.2MB/s]
Downloading best_ckpt_mp0.tar: 13%|█▎ | 1.10G/8.31G [00:24<02:14, 57.6MB/s]
Downloading best_ckpt_mp0.tar: 13%|█▎ | 1.11G/8.31G [00:24<02:15, 57.2MB/s]
Downloading best_ckpt_mp0.tar: 14%|█▎ | 1.12G/8.31G [00:24<02:12, 58.1MB/s]
Downloading best_ckpt_mp0.tar: 14%|█▎ | 1.13G/8.31G [00:24<02:49, 45.5MB/s]
Downloading best_ckpt_mp0.tar: 14%|█▍ | 1.14G/8.31G [00:24<02:37, 48.9MB/s]
Downloading best_ckpt_mp0.tar: 14%|█▍ | 1.15G/8.31G [00:25<02:49, 45.3MB/s]
Downloading best_ckpt_mp0.tar: 14%|█▍ | 1.16G/8.31G [00:25<02:41, 47.4MB/s]
Downloading best_ckpt_mp0.tar: 14%|█▍ | 1.17G/8.31G [00:25<02:34, 49.7MB/s]
Downloading best_ckpt_mp0.tar: 14%|█▍ | 1.18G/8.31G [00:25<02:22, 53.8MB/s]
Downloading best_ckpt_mp0.tar: 14%|█▍ | 1.19G/8.31G [00:26<02:55, 43.7MB/s]
Downloading best_ckpt_mp0.tar: 14%|█▍ | 1.20G/8.31G [00:26<02:48, 45.2MB/s]
Downloading best_ckpt_mp0.tar: 15%|█▍ | 1.21G/8.31G [00:26<02:33, 49.5MB/s]
Downloading best_ckpt_mp0.tar: 15%|█▍ | 1.22G/8.31G [00:26<02:27, 51.6MB/s]
Downloading best_ckpt_mp0.tar: 15%|█▍ | 1.23G/8.31G [00:26<02:25, 52.2MB/s]
Downloading best_ckpt_mp0.tar: 15%|█▍ | 1.24G/8.31G [00:27<02:23, 52.9MB/s]
Downloading best_ckpt_mp0.tar: 15%|█▌ | 1.25G/8.31G [00:27<02:51, 44.3MB/s]
Downloading best_ckpt_mp0.tar: 15%|█▌ | 1.26G/8.31G [00:27<02:40, 47.2MB/s]
Downloading best_ckpt_mp0.tar: 15%|█▌ | 1.27G/8.31G [00:27<02:27, 51.2MB/s]
Downloading best_ckpt_mp0.tar: 15%|█▌ | 1.28G/8.31G [00:27<02:18, 54.5MB/s]
Downloading best_ckpt_mp0.tar: 16%|█▌ | 1.29G/8.31G [00:28<02:11, 57.3MB/s]
Downloading best_ckpt_mp0.tar: 16%|█▌ | 1.30G/8.31G [00:28<02:01, 62.1MB/s]
Downloading best_ckpt_mp0.tar: 16%|█▌ | 1.31G/8.31G [00:28<02:00, 62.3MB/s]
Downloading best_ckpt_mp0.tar: 16%|█▌ | 1.32G/8.31G [00:28<02:42, 46.1MB/s]
Downloading best_ckpt_mp0.tar: 16%|█▌ | 1.33G/8.31G [00:28<02:36, 47.8MB/s]
Downloading best_ckpt_mp0.tar: 16%|█▌ | 1.34G/8.31G [00:29<02:26, 51.0MB/s]
Downloading best_ckpt_mp0.tar: 16%|█▌ | 1.35G/8.31G [00:29<02:30, 49.6MB/s]
Downloading best_ckpt_mp0.tar: 16%|█▋ | 1.36G/8.31G [00:29<02:35, 47.9MB/s]
Downloading best_ckpt_mp0.tar: 16%|█▋ | 1.37G/8.31G [00:29<02:26, 50.9MB/s]
Downloading best_ckpt_mp0.tar: 17%|█▋ | 1.38G/8.31G [00:30<02:56, 42.1MB/s]
Downloading best_ckpt_mp0.tar: 17%|█▋ | 1.39G/8.31G [00:30<02:32, 48.8MB/s]
Downloading best_ckpt_mp0.tar: 17%|█▋ | 1.40G/8.31G [00:30<02:26, 50.7MB/s]
Downloading best_ckpt_mp0.tar: 17%|█▋ | 1.41G/8.31G [00:30<02:12, 56.1MB/s]
Downloading best_ckpt_mp0.tar: 17%|█▋ | 1.42G/8.31G [00:30<01:56, 63.6MB/s]
Downloading best_ckpt_mp0.tar: 17%|█▋ | 1.43G/8.31G [00:30<01:49, 67.7MB/s]
Downloading best_ckpt_mp0.tar: 17%|█▋ | 1.44G/8.31G [00:31<01:49, 67.4MB/s]
Downloading best_ckpt_mp0.tar: 17%|█▋ | 1.45G/8.31G [00:31<02:12, 55.5MB/s]
Downloading best_ckpt_mp0.tar: 18%|█▊ | 1.46G/8.31G [00:31<02:56, 41.8MB/s]
Downloading best_ckpt_mp0.tar: 18%|█▊ | 1.46G/8.31G [00:31<02:27, 49.7MB/s]
Downloading best_ckpt_mp0.tar: 18%|█▊ | 1.47G/8.31G [00:31<02:07, 57.7MB/s]
Downloading best_ckpt_mp0.tar: 18%|█▊ | 1.48G/8.31G [00:32<01:52, 65.1MB/s]
Downloading best_ckpt_mp0.tar: 18%|█▊ | 1.49G/8.31G [00:32<01:42, 71.4MB/s]
Downloading best_ckpt_mp0.tar: 18%|█▊ | 1.50G/8.31G [00:32<01:55, 63.1MB/s]
Downloading best_ckpt_mp0.tar: 18%|█▊ | 1.51G/8.31G [00:32<01:59, 60.8MB/s]
Downloading best_ckpt_mp0.tar: 18%|█▊ | 1.52G/8.31G [00:32<01:58, 61.4MB/s]
Downloading best_ckpt_mp0.tar: 18%|█▊ | 1.53G/8.31G [00:32<02:02, 59.5MB/s]
Downloading best_ckpt_mp0.tar: 19%|█▊ | 1.54G/8.31G [00:33<02:05, 57.9MB/s]
Downloading best_ckpt_mp0.tar: 19%|█▊ | 1.55G/8.31G [00:33<02:07, 57.0MB/s]
Downloading best_ckpt_mp0.tar: 19%|█▉ | 1.56G/8.31G [00:33<02:32, 47.4MB/s]
Downloading best_ckpt_mp0.tar: 19%|█▉ | 1.57G/8.31G [00:33<02:17, 52.4MB/s]
Downloading best_ckpt_mp0.tar: 19%|█▉ | 1.58G/8.31G [00:33<02:06, 57.2MB/s]
Downloading best_ckpt_mp0.tar: 19%|█▉ | 1.59G/8.31G [00:33<01:54, 62.8MB/s]
Downloading best_ckpt_mp0.tar: 19%|█▉ | 1.60G/8.31G [00:34<02:05, 57.3MB/s]
Downloading best_ckpt_mp0.tar: 19%|█▉ | 1.61G/8.31G [00:34<02:06, 56.8MB/s]
Downloading best_ckpt_mp0.tar: 20%|█▉ | 1.62G/8.31G [00:34<02:02, 58.5MB/s]
Downloading best_ckpt_mp0.tar: 20%|█▉ | 1.63G/8.31G [00:34<02:40, 44.6MB/s]
Downloading best_ckpt_mp0.tar: 20%|█▉ | 1.64G/8.31G [00:35<02:38, 45.2MB/s]
Downloading best_ckpt_mp0.tar: 20%|█▉ | 1.65G/8.31G [00:35<02:34, 46.4MB/s]
Downloading best_ckpt_mp0.tar: 20%|█▉ | 1.66G/8.31G [00:35<02:32, 46.8MB/s]
Downloading best_ckpt_mp0.tar: 20%|██ | 1.67G/8.31G [00:35<02:26, 48.6MB/s]
Downloading best_ckpt_mp0.tar: 20%|██ | 1.68G/8.31G [00:35<02:19, 50.9MB/s]
Downloading best_ckpt_mp0.tar: 20%|██ | 1.69G/8.31G [00:36<02:55, 40.4MB/s]
Downloading best_ckpt_mp0.tar: 20%|██ | 1.70G/8.31G [00:36<02:36, 45.4MB/s]
Downloading best_ckpt_mp0.tar: 21%|██ | 1.71G/8.31G [00:36<02:28, 47.7MB/s]
Downloading best_ckpt_mp0.tar: 21%|██ | 1.72G/8.31G [00:36<02:23, 49.4MB/s]
Downloading best_ckpt_mp0.tar: 21%|██ | 1.73G/8.31G [00:37<02:23, 49.3MB/s]
Downloading best_ckpt_mp0.tar: 21%|██ | 1.74G/8.31G [00:37<02:16, 51.8MB/s]
Downloading best_ckpt_mp0.tar: 21%|██ | 1.75G/8.31G [00:37<02:21, 49.7MB/s]
Downloading best_ckpt_mp0.tar: 21%|██ | 1.76G/8.31G [00:37<02:43, 43.1MB/s]
Downloading best_ckpt_mp0.tar: 21%|██▏ | 1.77G/8.31G [00:38<02:31, 46.3MB/s]
Downloading best_ckpt_mp0.tar: 21%|██▏ | 1.78G/8.31G [00:38<02:21, 49.6MB/s]
Downloading best_ckpt_mp0.tar: 22%|██▏ | 1.79G/8.31G [00:38<02:23, 48.7MB/s]
Downloading best_ckpt_mp0.tar: 22%|██▏ | 1.80G/8.31G [00:38<02:32, 45.9MB/s]
Downloading best_ckpt_mp0.tar: 22%|██▏ | 1.81G/8.31G [00:38<02:36, 44.6MB/s]
Downloading best_ckpt_mp0.tar: 22%|██▏ | 1.82G/8.31G [00:39<02:49, 41.2MB/s]
Downloading best_ckpt_mp0.tar: 22%|██▏ | 1.83G/8.31G [00:39<02:40, 43.3MB/s]
Downloading best_ckpt_mp0.tar: 22%|██▏ | 1.84G/8.31G [00:39<02:51, 40.6MB/s]
Downloading best_ckpt_mp0.tar: 22%|██▏ | 1.85G/8.31G [00:40<03:00, 38.5MB/s]
Downloading best_ckpt_mp0.tar: 22%|██▏ | 1.86G/8.31G [00:40<02:48, 41.1MB/s]
Downloading best_ckpt_mp0.tar: 22%|██▏ | 1.87G/8.31G [00:40<02:41, 43.0MB/s]
Downloading best_ckpt_mp0.tar: 23%|██▎ | 1.88G/8.31G [00:40<03:17, 35.1MB/s]
Downloading best_ckpt_mp0.tar: 23%|██▎ | 1.88G/8.31G [00:41<03:08, 36.6MB/s]
Downloading best_ckpt_mp0.tar: 23%|██▎ | 1.89G/8.31G [00:41<02:55, 39.3MB/s]
Downloading best_ckpt_mp0.tar: 23%|██▎ | 1.90G/8.31G [00:41<02:34, 44.6MB/s]
Downloading best_ckpt_mp0.tar: 23%|██▎ | 1.91G/8.31G [00:41<02:13, 51.6MB/s]
Downloading best_ckpt_mp0.tar: 23%|██▎ | 1.92G/8.31G [00:41<02:08, 53.3MB/s]
Downloading best_ckpt_mp0.tar: 23%|██▎ | 1.93G/8.31G [00:42<02:00, 56.7MB/s]
Downloading best_ckpt_mp0.tar: 23%|██▎ | 1.94G/8.31G [00:42<02:12, 51.6MB/s]
Downloading best_ckpt_mp0.tar: 24%|██▎ | 1.95G/8.31G [00:42<02:01, 56.4MB/s]
Downloading best_ckpt_mp0.tar: 24%|██▎ | 1.96G/8.31G [00:42<02:01, 56.0MB/s]
Downloading best_ckpt_mp0.tar: 24%|██▎ | 1.97G/8.31G [00:42<02:01, 55.9MB/s]
Downloading best_ckpt_mp0.tar: 24%|██▍ | 1.98G/8.31G [00:42<01:58, 57.2MB/s]
Downloading best_ckpt_mp0.tar: 24%|██▍ | 1.99G/8.31G [00:43<02:05, 53.9MB/s]
Downloading best_ckpt_mp0.tar: 24%|██▍ | 2.00G/8.31G [00:43<02:41, 41.9MB/s]
Downloading best_ckpt_mp0.tar: 24%|██▍ | 2.01G/8.31G [00:43<02:34, 43.7MB/s]
Downloading best_ckpt_mp0.tar: 24%|██▍ | 2.02G/8.31G [00:43<02:17, 49.1MB/s]
Downloading best_ckpt_mp0.tar: 24%|██▍ | 2.03G/8.31G [00:44<02:09, 51.9MB/s]
Downloading best_ckpt_mp0.tar: 25%|██▍ | 2.04G/8.31G [00:44<02:09, 51.8MB/s]
Downloading best_ckpt_mp0.tar: 25%|██▍ | 2.05G/8.31G [00:44<02:03, 54.5MB/s]
Downloading best_ckpt_mp0.tar: 25%|██▍ | 2.06G/8.31G [00:44<01:54, 58.8MB/s]
Downloading best_ckpt_mp0.tar: 25%|██▍ | 2.07G/8.31G [00:45<02:30, 44.4MB/s]
Downloading best_ckpt_mp0.tar: 25%|██▌ | 2.08G/8.31G [00:45<02:17, 48.8MB/s]
Downloading best_ckpt_mp0.tar: 25%|██▌ | 2.09G/8.31G [00:45<02:08, 52.0MB/s]
Downloading best_ckpt_mp0.tar: 25%|██▌ | 2.10G/8.31G [00:45<01:51, 59.8MB/s]
Downloading best_ckpt_mp0.tar: 25%|██▌ | 2.11G/8.31G [00:45<01:48, 61.4MB/s]
Downloading best_ckpt_mp0.tar: 26%|██▌ | 2.12G/8.31G [00:45<01:38, 67.5MB/s]
Downloading best_ckpt_mp0.tar: 26%|██▌ | 2.13G/8.31G [00:45<01:57, 56.3MB/s]
Downloading best_ckpt_mp0.tar: 26%|██▌ | 2.14G/8.31G [00:46<01:45, 63.1MB/s]
Downloading best_ckpt_mp0.tar: 26%|██▌ | 2.15G/8.31G [00:46<01:40, 65.6MB/s]
Downloading best_ckpt_mp0.tar: 26%|██▌ | 2.16G/8.31G [00:46<01:38, 66.8MB/s]
Downloading best_ckpt_mp0.tar: 26%|██▌ | 2.17G/8.31G [00:46<01:41, 64.9MB/s]
Downloading best_ckpt_mp0.tar: 26%|██▌ | 2.18G/8.31G [00:46<01:42, 64.5MB/s]
Downloading best_ckpt_mp0.tar: 26%|██▋ | 2.19G/8.31G [00:47<02:13, 49.1MB/s]
Downloading best_ckpt_mp0.tar: 26%|██▋ | 2.20G/8.31G [00:47<02:07, 51.4MB/s]
Downloading best_ckpt_mp0.tar: 27%|██▋ | 2.21G/8.31G [00:47<01:56, 56.1MB/s]
Downloading best_ckpt_mp0.tar: 27%|██▋ | 2.22G/8.31G [00:47<01:52, 58.0MB/s]
Downloading best_ckpt_mp0.tar: 27%|██▋ | 2.23G/8.31G [00:47<01:45, 61.9MB/s]
Downloading best_ckpt_mp0.tar: 27%|██▋ | 2.24G/8.31G [00:47<01:49, 59.6MB/s]
Downloading best_ckpt_mp0.tar: 27%|██▋ | 2.25G/8.31G [00:48<01:52, 57.8MB/s]
Downloading best_ckpt_mp0.tar: 27%|██▋ | 2.26G/8.31G [00:48<02:07, 51.0MB/s]
Downloading best_ckpt_mp0.tar: 27%|██▋ | 2.27G/8.31G [00:48<01:56, 55.6MB/s]
Downloading best_ckpt_mp0.tar: 27%|██▋ | 2.28G/8.31G [00:48<01:46, 60.6MB/s]
Downloading best_ckpt_mp0.tar: 28%|██▊ | 2.29G/8.31G [00:48<01:42, 63.1MB/s]
Downloading best_ckpt_mp0.tar: 28%|██▊ | 2.29G/8.31G [00:48<01:43, 62.1MB/s]
Downloading best_ckpt_mp0.tar: 28%|██▊ | 2.30G/8.31G [00:49<01:33, 68.7MB/s]
Downloading best_ckpt_mp0.tar: 28%|██▊ | 2.31G/8.31G [00:49<01:55, 55.9MB/s]
Downloading best_ckpt_mp0.tar: 28%|██▊ | 2.32G/8.31G [00:49<01:46, 60.1MB/s]
Downloading best_ckpt_mp0.tar: 28%|██▊ | 2.33G/8.31G [00:49<02:03, 51.8MB/s]
Downloading best_ckpt_mp0.tar: 28%|██▊ | 2.34G/8.31G [00:49<02:00, 53.3MB/s]
Downloading best_ckpt_mp0.tar: 28%|██▊ | 2.35G/8.31G [00:50<01:47, 59.5MB/s]
Downloading best_ckpt_mp0.tar: 28%|██▊ | 2.36G/8.31G [00:50<01:38, 64.8MB/s]
Downloading best_ckpt_mp0.tar: 29%|██▊ | 2.37G/8.31G [00:50<01:33, 68.2MB/s]
Downloading best_ckpt_mp0.tar: 29%|██▊ | 2.38G/8.31G [00:50<01:50, 57.5MB/s]
Downloading best_ckpt_mp0.tar: 29%|██▉ | 2.39G/8.31G [00:50<01:45, 60.4MB/s]
Downloading best_ckpt_mp0.tar: 29%|██▉ | 2.40G/8.31G [00:50<01:43, 61.5MB/s]
Downloading best_ckpt_mp0.tar: 29%|██▉ | 2.41G/8.31G [00:51<01:39, 63.5MB/s]
Downloading best_ckpt_mp0.tar: 29%|██▉ | 2.42G/8.31G [00:51<01:45, 59.9MB/s]
Downloading best_ckpt_mp0.tar: 29%|██▉ | 2.43G/8.31G [00:51<01:43, 61.0MB/s]
Downloading best_ckpt_mp0.tar: 29%|██▉ | 2.44G/8.31G [00:51<02:16, 46.2MB/s]
Downloading best_ckpt_mp0.tar: 29%|██▉ | 2.45G/8.31G [00:51<02:00, 52.4MB/s]
Downloading best_ckpt_mp0.tar: 30%|██▉ | 2.46G/8.31G [00:52<01:52, 55.8MB/s]
Downloading best_ckpt_mp0.tar: 30%|██▉ | 2.47G/8.31G [00:52<01:44, 59.9MB/s]
Downloading best_ckpt_mp0.tar: 30%|██▉ | 2.48G/8.31G [00:52<01:43, 60.2MB/s]
Downloading best_ckpt_mp0.tar: 30%|██▉ | 2.49G/8.31G [00:52<01:39, 62.8MB/s]
Downloading best_ckpt_mp0.tar: 30%|███ | 2.50G/8.31G [00:52<02:08, 48.7MB/s]
Downloading best_ckpt_mp0.tar: 30%|███ | 2.51G/8.31G [00:53<02:06, 49.4MB/s]
Downloading best_ckpt_mp0.tar: 30%|███ | 2.52G/8.31G [00:53<02:00, 51.7MB/s]
Downloading best_ckpt_mp0.tar: 30%|███ | 2.53G/8.31G [00:53<01:49, 56.6MB/s]
Downloading best_ckpt_mp0.tar: 31%|███ | 2.54G/8.31G [00:53<01:48, 57.0MB/s]
Downloading best_ckpt_mp0.tar: 31%|███ | 2.55G/8.31G [00:53<01:44, 59.0MB/s]
Downloading best_ckpt_mp0.tar: 31%|███ | 2.56G/8.31G [00:53<01:38, 62.9MB/s]
Downloading best_ckpt_mp0.tar: 31%|███ | 2.57G/8.31G [00:54<02:04, 49.7MB/s]
Downloading best_ckpt_mp0.tar: 31%|███ | 2.58G/8.31G [00:54<01:55, 53.2MB/s]
Downloading best_ckpt_mp0.tar: 31%|███ | 2.59G/8.31G [00:54<01:41, 60.3MB/s]
Downloading best_ckpt_mp0.tar: 31%|███▏ | 2.60G/8.31G [00:54<01:35, 64.1MB/s]
Downloading best_ckpt_mp0.tar: 31%|███▏ | 2.61G/8.31G [00:54<01:38, 61.9MB/s]
Downloading best_ckpt_mp0.tar: 31%|███▏ | 2.62G/8.31G [00:54<01:39, 61.4MB/s]
Downloading best_ckpt_mp0.tar: 32%|███▏ | 2.63G/8.31G [00:55<01:54, 53.4MB/s]
Downloading best_ckpt_mp0.tar: 32%|███▏ | 2.64G/8.31G [00:55<01:41, 59.9MB/s]
Downloading best_ckpt_mp0.tar: 32%|███▏ | 2.65G/8.31G [00:55<01:36, 63.1MB/s]
Downloading best_ckpt_mp0.tar: 32%|███▏ | 2.66G/8.31G [00:55<01:28, 68.5MB/s]
Downloading best_ckpt_mp0.tar: 32%|███▏ | 2.67G/8.31G [00:55<01:36, 62.5MB/s]
Downloading best_ckpt_mp0.tar: 32%|███▏ | 2.68G/8.31G [00:55<01:27, 68.8MB/s]
Downloading best_ckpt_mp0.tar: 32%|███▏ | 2.69G/8.31G [00:56<01:41, 59.6MB/s]
Downloading best_ckpt_mp0.tar: 32%|███▏ | 2.70G/8.31G [00:56<01:32, 65.3MB/s]
Downloading best_ckpt_mp0.tar: 33%|███▎ | 2.71G/8.31G [00:56<01:35, 62.9MB/s]
Downloading best_ckpt_mp0.tar: 33%|███▎ | 2.71G/8.31G [00:56<01:28, 67.6MB/s]
Downloading best_ckpt_mp0.tar: 33%|███▎ | 2.72G/8.31G [00:56<01:29, 67.4MB/s]
Downloading best_ckpt_mp0.tar: 33%|███▎ | 2.73G/8.31G [00:56<01:24, 70.7MB/s]
Downloading best_ckpt_mp0.tar: 33%|███▎ | 2.74G/8.31G [00:57<01:20, 74.6MB/s]
Downloading best_ckpt_mp0.tar: 33%|███▎ | 2.75G/8.31G [00:57<01:34, 63.0MB/s]
Downloading best_ckpt_mp0.tar: 33%|███▎ | 2.76G/8.31G [00:57<01:26, 68.7MB/s]
Downloading best_ckpt_mp0.tar: 33%|███▎ | 2.77G/8.31G [00:57<01:20, 73.5MB/s]
Downloading best_ckpt_mp0.tar: 33%|███▎ | 2.78G/8.31G [00:57<01:17, 76.8MB/s]
Downloading best_ckpt_mp0.tar: 34%|███▎ | 2.79G/8.31G [00:57<01:14, 79.8MB/s]
Downloading best_ckpt_mp0.tar: 34%|███▎ | 2.80G/8.31G [00:57<01:12, 81.5MB/s]
Downloading best_ckpt_mp0.tar: 34%|███▍ | 2.81G/8.31G [00:58<01:19, 73.9MB/s]
Downloading best_ckpt_mp0.tar: 34%|███▍ | 2.82G/8.31G [00:58<01:26, 68.4MB/s]
Downloading best_ckpt_mp0.tar: 34%|███▍ | 2.83G/8.31G [00:58<01:41, 58.1MB/s]
Downloading best_ckpt_mp0.tar: 34%|███▍ | 2.84G/8.31G [00:58<01:37, 60.3MB/s]
Downloading best_ckpt_mp0.tar: 34%|███▍ | 2.85G/8.31G [00:58<01:32, 63.6MB/s]
Downloading best_ckpt_mp0.tar: 34%|███▍ | 2.86G/8.31G [00:58<01:25, 68.0MB/s]
Downloading best_ckpt_mp0.tar: 35%|███▍ | 2.87G/8.31G [00:59<02:04, 47.0MB/s]
Downloading best_ckpt_mp0.tar: 35%|███▍ | 2.88G/8.31G [00:59<02:01, 47.8MB/s]
Downloading best_ckpt_mp0.tar: 35%|███▍ | 2.89G/8.31G [00:59<01:46, 54.4MB/s]
Downloading best_ckpt_mp0.tar: 35%|███▍ | 2.90G/8.31G [00:59<01:38, 58.8MB/s]
Downloading best_ckpt_mp0.tar: 35%|███▌ | 2.91G/8.31G [00:59<01:29, 64.6MB/s]
Downloading best_ckpt_mp0.tar: 35%|███▌ | 2.92G/8.31G [01:00<01:27, 66.1MB/s]
Downloading best_ckpt_mp0.tar: 35%|███▌ | 2.93G/8.31G [01:00<01:23, 69.4MB/s]
Downloading best_ckpt_mp0.tar: 35%|███▌ | 2.94G/8.31G [01:00<01:45, 54.6MB/s]
Downloading best_ckpt_mp0.tar: 35%|███▌ | 2.95G/8.31G [01:00<01:33, 61.4MB/s]
Downloading best_ckpt_mp0.tar: 36%|███▌ | 2.96G/8.31G [01:00<01:29, 64.2MB/s]
Downloading best_ckpt_mp0.tar: 36%|███▌ | 2.97G/8.31G [01:00<01:26, 66.6MB/s]
Downloading best_ckpt_mp0.tar: 36%|███▌ | 2.98G/8.31G [01:01<01:40, 57.1MB/s]
Downloading best_ckpt_mp0.tar: 36%|███▌ | 2.99G/8.31G [01:01<01:49, 52.4MB/s]
Downloading best_ckpt_mp0.tar: 36%|███▌ | 3.00G/8.31G [01:01<02:37, 36.3MB/s]
Downloading best_ckpt_mp0.tar: 36%|███▌ | 3.01G/8.31G [01:01<02:13, 42.5MB/s]
Downloading best_ckpt_mp0.tar: 36%|███▋ | 3.02G/8.31G [01:02<02:03, 45.8MB/s]
Downloading best_ckpt_mp0.tar: 36%|███▋ | 3.03G/8.31G [01:02<01:46, 53.3MB/s]
Downloading best_ckpt_mp0.tar: 37%|███▋ | 3.04G/8.31G [01:02<01:52, 50.1MB/s]
Downloading best_ckpt_mp0.tar: 37%|███▋ | 3.05G/8.31G [01:02<01:45, 53.4MB/s]
Downloading best_ckpt_mp0.tar: 37%|███▋ | 3.06G/8.31G [01:02<01:38, 57.0MB/s]
Downloading best_ckpt_mp0.tar: 37%|███▋ | 3.07G/8.31G [01:03<02:11, 42.7MB/s]
Downloading best_ckpt_mp0.tar: 37%|███▋ | 3.08G/8.31G [01:03<02:09, 43.5MB/s]
Downloading best_ckpt_mp0.tar: 37%|███▋ | 3.09G/8.31G [01:03<01:58, 47.2MB/s]
Downloading best_ckpt_mp0.tar: 37%|███▋ | 3.10G/8.31G [01:03<01:45, 53.1MB/s]
Downloading best_ckpt_mp0.tar: 37%|███▋ | 3.11G/8.31G [01:03<01:44, 53.5MB/s]
Downloading best_ckpt_mp0.tar: 37%|███▋ | 3.12G/8.31G [01:04<01:49, 50.9MB/s]
Downloading best_ckpt_mp0.tar: 38%|███▊ | 3.12G/8.31G [01:04<02:12, 41.9MB/s]
Downloading best_ckpt_mp0.tar: 38%|███▊ | 3.13G/8.31G [01:04<01:58, 46.9MB/s]
Downloading best_ckpt_mp0.tar: 38%|███▊ | 3.14G/8.31G [01:04<01:56, 47.7MB/s]
Downloading best_ckpt_mp0.tar: 38%|███▊ | 3.15G/8.31G [01:05<01:46, 52.0MB/s]
Downloading best_ckpt_mp0.tar: 38%|███▊ | 3.16G/8.31G [01:05<01:41, 54.6MB/s]
Downloading best_ckpt_mp0.tar: 38%|███▊ | 3.17G/8.31G [01:05<01:31, 60.4MB/s]
Downloading best_ckpt_mp0.tar: 38%|███▊ | 3.18G/8.31G [01:05<01:25, 64.0MB/s]
Downloading best_ckpt_mp0.tar: 38%|███▊ | 3.19G/8.31G [01:05<01:51, 49.2MB/s]
Downloading best_ckpt_mp0.tar: 39%|███▊ | 3.20G/8.31G [01:06<01:48, 50.7MB/s]
Downloading best_ckpt_mp0.tar: 39%|███▊ | 3.21G/8.31G [01:06<01:46, 51.4MB/s]
Downloading best_ckpt_mp0.tar: 39%|███▉ | 3.22G/8.31G [01:06<01:40, 54.4MB/s]
Downloading best_ckpt_mp0.tar: 39%|███▉ | 3.23G/8.31G [01:06<01:39, 54.8MB/s]
Downloading best_ckpt_mp0.tar: 39%|███▉ | 3.24G/8.31G [01:06<01:35, 56.8MB/s]
Downloading best_ckpt_mp0.tar: 39%|███▉ | 3.25G/8.31G [01:07<01:56, 46.5MB/s]
Downloading best_ckpt_mp0.tar: 39%|███▉ | 3.26G/8.31G [01:07<01:47, 50.3MB/s]
Downloading best_ckpt_mp0.tar: 39%|███▉ | 3.27G/8.31G [01:07<01:42, 52.7MB/s]
Downloading best_ckpt_mp0.tar: 39%|███▉ | 3.28G/8.31G [01:07<01:37, 55.6MB/s]
Downloading best_ckpt_mp0.tar: 40%|███▉ | 3.29G/8.31G [01:07<01:36, 55.7MB/s]
Downloading best_ckpt_mp0.tar: 40%|███▉ | 3.30G/8.31G [01:08<01:53, 47.4MB/s]
Downloading best_ckpt_mp0.tar: 40%|███▉ | 3.31G/8.31G [01:08<02:06, 42.5MB/s]
Downloading best_ckpt_mp0.tar: 40%|███▉ | 3.32G/8.31G [01:08<01:56, 46.1MB/s]
Downloading best_ckpt_mp0.tar: 40%|████ | 3.33G/8.31G [01:08<01:49, 49.0MB/s]
Downloading best_ckpt_mp0.tar: 40%|████ | 3.34G/8.31G [01:08<01:47, 49.5MB/s]
Downloading best_ckpt_mp0.tar: 40%|████ | 3.35G/8.31G [01:09<01:49, 48.8MB/s]
Downloading best_ckpt_mp0.tar: 40%|████ | 3.36G/8.31G [01:09<01:42, 51.6MB/s]
Downloading best_ckpt_mp0.tar: 41%|████ | 3.37G/8.31G [01:09<01:40, 53.0MB/s]
Downloading best_ckpt_mp0.tar: 41%|████ | 3.38G/8.31G [01:10<02:24, 36.6MB/s]
Downloading best_ckpt_mp0.tar: 41%|████ | 3.39G/8.31G [01:10<01:58, 44.6MB/s]
Downloading best_ckpt_mp0.tar: 41%|████ | 3.40G/8.31G [01:10<01:39, 52.8MB/s]
Downloading best_ckpt_mp0.tar: 41%|████ | 3.41G/8.31G [01:10<01:32, 57.0MB/s]
Downloading best_ckpt_mp0.tar: 41%|████ | 3.42G/8.31G [01:10<01:30, 58.2MB/s]
Downloading best_ckpt_mp0.tar: 41%|████▏ | 3.43G/8.31G [01:10<01:30, 57.8MB/s]
Downloading best_ckpt_mp0.tar: 41%|████▏ | 3.44G/8.31G [01:11<01:46, 49.0MB/s]
Downloading best_ckpt_mp0.tar: 41%|████▏ | 3.45G/8.31G [01:11<01:37, 53.5MB/s]
Downloading best_ckpt_mp0.tar: 42%|████▏ | 3.46G/8.31G [01:11<01:47, 48.6MB/s]
Downloading best_ckpt_mp0.tar: 42%|████▏ | 3.47G/8.31G [01:11<01:47, 48.4MB/s]
Downloading best_ckpt_mp0.tar: 42%|████▏ | 3.48G/8.31G [01:11<01:44, 49.8MB/s]
Downloading best_ckpt_mp0.tar: 42%|████▏ | 3.49G/8.31G [01:12<01:42, 50.8MB/s]
Downloading best_ckpt_mp0.tar: 42%|████▏ | 3.50G/8.31G [01:12<01:46, 48.5MB/s]
Downloading best_ckpt_mp0.tar: 42%|████▏ | 3.51G/8.31G [01:12<01:48, 47.5MB/s]
Downloading best_ckpt_mp0.tar: 42%|████▏ | 3.52G/8.31G [01:12<02:02, 42.1MB/s]
Downloading best_ckpt_mp0.tar: 42%|████▏ | 3.53G/8.31G [01:13<01:47, 47.7MB/s]
Downloading best_ckpt_mp0.tar: 43%|████▎ | 3.54G/8.31G [01:13<02:10, 39.4MB/s]
Downloading best_ckpt_mp0.tar: 43%|████▎ | 3.54G/8.31G [01:13<01:58, 43.1MB/s]
Downloading best_ckpt_mp0.tar: 43%|████▎ | 3.55G/8.31G [01:13<02:09, 39.3MB/s]
Downloading best_ckpt_mp0.tar: 43%|████▎ | 3.56G/8.31G [01:14<02:09, 39.3MB/s]
Downloading best_ckpt_mp0.tar: 43%|████▎ | 3.57G/8.31G [01:14<01:59, 42.6MB/s]
Downloading best_ckpt_mp0.tar: 43%|████▎ | 3.58G/8.31G [01:14<02:29, 33.9MB/s]
Downloading best_ckpt_mp0.tar: 43%|████▎ | 3.59G/8.31G [01:14<02:02, 41.4MB/s]
Downloading best_ckpt_mp0.tar: 43%|████▎ | 3.60G/8.31G [01:15<01:59, 42.2MB/s]
Downloading best_ckpt_mp0.tar: 43%|████▎ | 3.61G/8.31G [01:15<01:44, 48.1MB/s]
Downloading best_ckpt_mp0.tar: 44%|████▎ | 3.62G/8.31G [01:15<02:01, 41.3MB/s]
Downloading best_ckpt_mp0.tar: 44%|████▎ | 3.63G/8.31G [01:15<02:00, 41.5MB/s]
Downloading best_ckpt_mp0.tar: 44%|████▍ | 3.64G/8.31G [01:16<01:58, 42.5MB/s]
Downloading best_ckpt_mp0.tar: 44%|████▍ | 3.65G/8.31G [01:16<01:42, 48.6MB/s]
Downloading best_ckpt_mp0.tar: 44%|████▍ | 3.66G/8.31G [01:16<02:28, 33.6MB/s]
Downloading best_ckpt_mp0.tar: 44%|████▍ | 3.67G/8.31G [01:17<02:09, 38.3MB/s]
Downloading best_ckpt_mp0.tar: 44%|████▍ | 3.68G/8.31G [01:17<01:48, 45.9MB/s]
Downloading best_ckpt_mp0.tar: 44%|████▍ | 3.69G/8.31G [01:17<01:57, 42.3MB/s]
Downloading best_ckpt_mp0.tar: 45%|████▍ | 3.70G/8.31G [01:17<01:41, 48.7MB/s]
Downloading best_ckpt_mp0.tar: 45%|████▍ | 3.71G/8.31G [01:17<01:42, 47.9MB/s]
Downloading best_ckpt_mp0.tar: 45%|████▍ | 3.72G/8.31G [01:18<01:50, 44.5MB/s]
Downloading best_ckpt_mp0.tar: 45%|████▍ | 3.73G/8.31G [01:18<01:34, 51.9MB/s]
Downloading best_ckpt_mp0.tar: 45%|████▌ | 3.74G/8.31G [01:18<02:05, 39.1MB/s]
Downloading best_ckpt_mp0.tar: 45%|████▌ | 3.75G/8.31G [01:18<02:12, 36.9MB/s]
Downloading best_ckpt_mp0.tar: 45%|████▌ | 3.76G/8.31G [01:19<02:04, 39.3MB/s]
Downloading best_ckpt_mp0.tar: 45%|████▌ | 3.77G/8.31G [01:19<01:52, 43.4MB/s]
Downloading best_ckpt_mp0.tar: 45%|████▌ | 3.78G/8.31G [01:19<01:40, 48.4MB/s]
Downloading best_ckpt_mp0.tar: 46%|████▌ | 3.79G/8.31G [01:19<01:45, 46.2MB/s]
Downloading best_ckpt_mp0.tar: 46%|████▌ | 3.80G/8.31G [01:19<01:36, 50.0MB/s]
Downloading best_ckpt_mp0.tar: 46%|████▌ | 3.81G/8.31G [01:20<01:28, 54.8MB/s]
Downloading best_ckpt_mp0.tar: 46%|████▌ | 3.82G/8.31G [01:20<01:40, 48.0MB/s]
Downloading best_ckpt_mp0.tar: 46%|████▌ | 3.83G/8.31G [01:20<01:26, 55.8MB/s]
Downloading best_ckpt_mp0.tar: 46%|████▌ | 3.84G/8.31G [01:20<01:18, 61.5MB/s]
Downloading best_ckpt_mp0.tar: 46%|████▋ | 3.85G/8.31G [01:20<01:29, 53.8MB/s]
Downloading best_ckpt_mp0.tar: 46%|████▋ | 3.86G/8.31G [01:21<01:39, 47.8MB/s]
Downloading best_ckpt_mp0.tar: 47%|████▋ | 3.87G/8.31G [01:21<01:27, 54.3MB/s]
Downloading best_ckpt_mp0.tar: 47%|████▋ | 3.88G/8.31G [01:21<01:39, 48.0MB/s]
Downloading best_ckpt_mp0.tar: 47%|████▋ | 3.89G/8.31G [01:21<01:25, 55.2MB/s]
Downloading best_ckpt_mp0.tar: 47%|████▋ | 3.90G/8.31G [01:21<01:20, 59.0MB/s]
Downloading best_ckpt_mp0.tar: 47%|████▋ | 3.91G/8.31G [01:21<01:16, 61.7MB/s]
Downloading best_ckpt_mp0.tar: 47%|████▋ | 3.92G/8.31G [01:22<01:10, 67.4MB/s]
Downloading best_ckpt_mp0.tar: 47%|████▋ | 3.93G/8.31G [01:22<01:05, 72.0MB/s]
Downloading best_ckpt_mp0.tar: 47%|████▋ | 3.94G/8.31G [01:22<01:26, 54.0MB/s]
Downloading best_ckpt_mp0.tar: 47%|████▋ | 3.95G/8.31G [01:22<01:25, 55.1MB/s]
Downloading best_ckpt_mp0.tar: 48%|████▊ | 3.96G/8.31G [01:22<01:23, 56.0MB/s]
Downloading best_ckpt_mp0.tar: 48%|████▊ | 3.96G/8.31G [01:23<01:18, 59.6MB/s]
Downloading best_ckpt_mp0.tar: 48%|████▊ | 3.97G/8.31G [01:23<01:16, 60.5MB/s]
Downloading best_ckpt_mp0.tar: 48%|████▊ | 3.98G/8.31G [01:23<01:17, 59.8MB/s]
Downloading best_ckpt_mp0.tar: 48%|████▊ | 3.99G/8.31G [01:23<01:21, 56.8MB/s]
Downloading best_ckpt_mp0.tar: 48%|████▊ | 4.00G/8.31G [01:23<01:44, 44.3MB/s]
Downloading best_ckpt_mp0.tar: 48%|████▊ | 4.01G/8.31G [01:24<01:32, 49.8MB/s]
Downloading best_ckpt_mp0.tar: 48%|████▊ | 4.02G/8.31G [01:24<01:31, 50.1MB/s]
Downloading best_ckpt_mp0.tar: 49%|████▊ | 4.03G/8.31G [01:24<01:37, 47.2MB/s]
Downloading best_ckpt_mp0.tar: 49%|████▊ | 4.04G/8.31G [01:24<01:28, 51.8MB/s]
Downloading best_ckpt_mp0.tar: 49%|████▉ | 4.05G/8.31G [01:24<01:17, 58.9MB/s]
Downloading best_ckpt_mp0.tar: 49%|████▉ | 4.06G/8.31G [01:25<01:25, 53.5MB/s]
Downloading best_ckpt_mp0.tar: 49%|████▉ | 4.07G/8.31G [01:25<01:24, 53.8MB/s]
Downloading best_ckpt_mp0.tar: 49%|████▉ | 4.08G/8.31G [01:25<01:16, 59.5MB/s]
Downloading best_ckpt_mp0.tar: 49%|████▉ | 4.09G/8.31G [01:25<01:11, 63.5MB/s]
Downloading best_ckpt_mp0.tar: 49%|████▉ | 4.10G/8.31G [01:25<01:13, 61.4MB/s]
Downloading best_ckpt_mp0.tar: 49%|████▉ | 4.11G/8.31G [01:25<01:08, 66.0MB/s]
Downloading best_ckpt_mp0.tar: 50%|████▉ | 4.12G/8.31G [01:25<01:02, 72.2MB/s]
Downloading best_ckpt_mp0.tar: 50%|████▉ | 4.13G/8.31G [01:26<01:15, 59.7MB/s]
Downloading best_ckpt_mp0.tar: 50%|████▉ | 4.14G/8.31G [01:26<01:28, 50.3MB/s]
Downloading best_ckpt_mp0.tar: 50%|████▉ | 4.15G/8.31G [01:26<01:28, 50.4MB/s]
Downloading best_ckpt_mp0.tar: 50%|█████ | 4.16G/8.31G [01:26<01:26, 51.7MB/s]
Downloading best_ckpt_mp0.tar: 50%|█████ | 4.17G/8.31G [01:27<01:19, 55.9MB/s]
Downloading best_ckpt_mp0.tar: 50%|█████ | 4.18G/8.31G [01:27<01:13, 60.0MB/s]
Downloading best_ckpt_mp0.tar: 50%|█████ | 4.19G/8.31G [01:27<01:25, 52.0MB/s]
Downloading best_ckpt_mp0.tar: 51%|█████ | 4.20G/8.31G [01:27<01:14, 59.1MB/s]
Downloading best_ckpt_mp0.tar: 51%|█████ | 4.21G/8.31G [01:27<01:12, 60.8MB/s]
Downloading best_ckpt_mp0.tar: 51%|█████ | 4.22G/8.31G [01:27<01:14, 59.3MB/s]
Downloading best_ckpt_mp0.tar: 51%|█████ | 4.23G/8.31G [01:28<01:13, 59.6MB/s]
Downloading best_ckpt_mp0.tar: 51%|█████ | 4.24G/8.31G [01:28<01:05, 66.6MB/s]
Downloading best_ckpt_mp0.tar: 51%|█████ | 4.25G/8.31G [01:28<01:11, 60.8MB/s]
Downloading best_ckpt_mp0.tar: 51%|█████ | 4.26G/8.31G [01:28<01:54, 38.0MB/s]
Downloading best_ckpt_mp0.tar: 51%|█████▏ | 4.27G/8.31G [01:29<01:47, 40.2MB/s]
Downloading best_ckpt_mp0.tar: 51%|█████▏ | 4.28G/8.31G [01:29<01:37, 44.6MB/s]
Downloading best_ckpt_mp0.tar: 52%|█████▏ | 4.29G/8.31G [01:29<01:40, 42.9MB/s]
Downloading best_ckpt_mp0.tar: 52%|█████▏ | 4.30G/8.31G [01:29<01:27, 49.4MB/s]
Downloading best_ckpt_mp0.tar: 52%|█████▏ | 4.31G/8.31G [01:30<01:29, 47.8MB/s]
Downloading best_ckpt_mp0.tar: 52%|█████▏ | 4.32G/8.31G [01:30<01:38, 43.4MB/s]
Downloading best_ckpt_mp0.tar: 52%|█████▏ | 4.33G/8.31G [01:30<01:28, 48.4MB/s]
Downloading best_ckpt_mp0.tar: 52%|█████▏ | 4.34G/8.31G [01:30<01:20, 52.7MB/s]
Downloading best_ckpt_mp0.tar: 52%|█████▏ | 4.35G/8.31G [01:30<01:21, 52.1MB/s]
Downloading best_ckpt_mp0.tar: 52%|█████▏ | 4.36G/8.31G [01:30<01:12, 58.3MB/s]
Downloading best_ckpt_mp0.tar: 53%|█████▎ | 4.37G/8.31G [01:31<01:34, 44.8MB/s]
Downloading best_ckpt_mp0.tar: 53%|█████▎ | 4.38G/8.31G [01:31<01:19, 53.0MB/s]
Downloading best_ckpt_mp0.tar: 53%|█████▎ | 4.38G/8.31G [01:31<01:09, 60.9MB/s]
Downloading best_ckpt_mp0.tar: 53%|█████▎ | 4.39G/8.31G [01:31<01:01, 67.9MB/s]
Downloading best_ckpt_mp0.tar: 53%|█████▎ | 4.40G/8.31G [01:31<00:57, 73.4MB/s]
Downloading best_ckpt_mp0.tar: 53%|█████▎ | 4.41G/8.31G [01:32<01:20, 51.9MB/s]
Downloading best_ckpt_mp0.tar: 53%|█████▎ | 4.42G/8.31G [01:32<01:15, 55.0MB/s]
Downloading best_ckpt_mp0.tar: 53%|█████▎ | 4.43G/8.31G [01:32<01:15, 54.9MB/s]
Downloading best_ckpt_mp0.tar: 53%|█████▎ | 4.44G/8.31G [01:32<01:25, 48.7MB/s]
Downloading best_ckpt_mp0.tar: 54%|█████▎ | 4.45G/8.31G [01:32<01:19, 51.8MB/s]
Downloading best_ckpt_mp0.tar: 54%|█████▎ | 4.46G/8.31G [01:33<01:14, 55.6MB/s]
Downloading best_ckpt_mp0.tar: 54%|█████▍ | 4.47G/8.31G [01:33<01:10, 58.5MB/s]
Downloading best_ckpt_mp0.tar: 54%|█████▍ | 4.48G/8.31G [01:33<01:03, 64.4MB/s]
Downloading best_ckpt_mp0.tar: 54%|█████▍ | 4.49G/8.31G [01:33<00:58, 70.6MB/s]
Downloading best_ckpt_mp0.tar: 54%|█████▍ | 4.50G/8.31G [01:33<01:05, 62.0MB/s]
Downloading best_ckpt_mp0.tar: 54%|█████▍ | 4.51G/8.31G [01:33<01:05, 62.6MB/s]
Downloading best_ckpt_mp0.tar: 54%|█████▍ | 4.52G/8.31G [01:33<01:01, 65.9MB/s]
Downloading best_ckpt_mp0.tar: 55%|█████▍ | 4.53G/8.31G [01:34<00:59, 67.7MB/s]
Downloading best_ckpt_mp0.tar: 55%|█████▍ | 4.54G/8.31G [01:34<01:11, 56.3MB/s]
Downloading best_ckpt_mp0.tar: 55%|█████▍ | 4.55G/8.31G [01:34<01:07, 59.4MB/s]
Downloading best_ckpt_mp0.tar: 55%|█████▍ | 4.56G/8.31G [01:34<01:17, 51.8MB/s]
Downloading best_ckpt_mp0.tar: 55%|█████▌ | 4.57G/8.31G [01:34<01:07, 59.9MB/s]
Downloading best_ckpt_mp0.tar: 55%|█████▌ | 4.58G/8.31G [01:35<01:04, 62.5MB/s]
Downloading best_ckpt_mp0.tar: 55%|█████▌ | 4.59G/8.31G [01:35<00:59, 67.3MB/s]
Downloading best_ckpt_mp0.tar: 55%|█████▌ | 4.60G/8.31G [01:35<00:56, 70.4MB/s]
Downloading best_ckpt_mp0.tar: 55%|█████▌ | 4.61G/8.31G [01:35<00:59, 66.7MB/s]
Downloading best_ckpt_mp0.tar: 56%|█████▌ | 4.62G/8.31G [01:35<01:01, 64.6MB/s]
Downloading best_ckpt_mp0.tar: 56%|█████▌ | 4.63G/8.31G [01:35<01:16, 52.0MB/s]
Downloading best_ckpt_mp0.tar: 56%|█████▌ | 4.64G/8.31G [01:36<01:10, 56.1MB/s]
Downloading best_ckpt_mp0.tar: 56%|█████▌ | 4.65G/8.31G [01:36<01:03, 62.1MB/s]
Downloading best_ckpt_mp0.tar: 56%|█████▌ | 4.66G/8.31G [01:36<01:03, 62.2MB/s]
Downloading best_ckpt_mp0.tar: 56%|█████▌ | 4.67G/8.31G [01:36<00:57, 68.2MB/s]
Downloading best_ckpt_mp0.tar: 56%|█████▋ | 4.68G/8.31G [01:36<00:55, 69.8MB/s]
Downloading best_ckpt_mp0.tar: 56%|█████▋ | 4.69G/8.31G [01:36<01:07, 58.0MB/s]
Downloading best_ckpt_mp0.tar: 57%|█████▋ | 4.70G/8.31G [01:37<01:02, 62.0MB/s]
Downloading best_ckpt_mp0.tar: 57%|█████▋ | 4.71G/8.31G [01:37<01:00, 63.8MB/s]
Downloading best_ckpt_mp0.tar: 57%|█████▋ | 4.72G/8.31G [01:37<01:03, 60.6MB/s]
Downloading best_ckpt_mp0.tar: 57%|█████▋ | 4.73G/8.31G [01:37<01:05, 58.4MB/s]
Downloading best_ckpt_mp0.tar: 57%|█████▋ | 4.74G/8.31G [01:37<01:02, 61.7MB/s]
Downloading best_ckpt_mp0.tar: 57%|█████▋ | 4.75G/8.31G [01:37<01:02, 61.3MB/s]
Downloading best_ckpt_mp0.tar: 57%|█████▋ | 4.76G/8.31G [01:38<01:17, 49.0MB/s]
Downloading best_ckpt_mp0.tar: 57%|█████▋ | 4.77G/8.31G [01:38<01:26, 44.0MB/s]
Downloading best_ckpt_mp0.tar: 57%|█████▋ | 4.78G/8.31G [01:38<01:21, 46.6MB/s]
Downloading best_ckpt_mp0.tar: 58%|█████▊ | 4.79G/8.31G [01:39<01:32, 40.8MB/s]
Downloading best_ckpt_mp0.tar: 58%|█████▊ | 4.79G/8.31G [01:39<01:24, 44.8MB/s]
Downloading best_ckpt_mp0.tar: 58%|█████▊ | 4.80G/8.31G [01:39<01:31, 40.9MB/s]
Downloading best_ckpt_mp0.tar: 58%|█████▊ | 4.81G/8.31G [01:39<01:34, 39.6MB/s]
Downloading best_ckpt_mp0.tar: 58%|█████▊ | 4.82G/8.31G [01:40<01:27, 42.8MB/s]
Downloading best_ckpt_mp0.tar: 58%|█████▊ | 4.83G/8.31G [01:40<01:27, 42.5MB/s]
Downloading best_ckpt_mp0.tar: 58%|█████▊ | 4.84G/8.31G [01:40<01:24, 43.9MB/s]
Downloading best_ckpt_mp0.tar: 58%|█████▊ | 4.85G/8.31G [01:40<01:16, 48.4MB/s]
Downloading best_ckpt_mp0.tar: 59%|█████▊ | 4.86G/8.31G [01:40<01:09, 53.0MB/s]
Downloading best_ckpt_mp0.tar: 59%|█████▊ | 4.87G/8.31G [01:41<01:27, 42.2MB/s]
Downloading best_ckpt_mp0.tar: 59%|█████▉ | 4.88G/8.31G [01:41<01:23, 44.2MB/s]
Downloading best_ckpt_mp0.tar: 59%|█████▉ | 4.89G/8.31G [01:41<01:20, 45.4MB/s]
Downloading best_ckpt_mp0.tar: 59%|█████▉ | 4.90G/8.31G [01:41<01:14, 49.3MB/s]
Downloading best_ckpt_mp0.tar: 59%|█████▉ | 4.91G/8.31G [01:42<01:21, 44.5MB/s]
Downloading best_ckpt_mp0.tar: 59%|█████▉ | 4.92G/8.31G [01:42<01:12, 49.9MB/s]
Downloading best_ckpt_mp0.tar: 59%|█████▉ | 4.93G/8.31G [01:42<01:18, 46.0MB/s]
Downloading best_ckpt_mp0.tar: 59%|█████▉ | 4.94G/8.31G [01:42<01:31, 39.3MB/s]
Downloading best_ckpt_mp0.tar: 60%|█████▉ | 4.95G/8.31G [01:43<01:27, 41.2MB/s]
Downloading best_ckpt_mp0.tar: 60%|█████▉ | 4.96G/8.31G [01:43<01:17, 46.2MB/s]
Downloading best_ckpt_mp0.tar: 60%|█████▉ | 4.97G/8.31G [01:43<01:15, 47.5MB/s]
Downloading best_ckpt_mp0.tar: 60%|█████▉ | 4.98G/8.31G [01:43<01:10, 50.6MB/s]
Downloading best_ckpt_mp0.tar: 60%|██████ | 4.99G/8.31G [01:43<01:03, 55.9MB/s]
Downloading best_ckpt_mp0.tar: 60%|██████ | 5.00G/8.31G [01:44<01:09, 51.1MB/s]
Downloading best_ckpt_mp0.tar: 60%|██████ | 5.01G/8.31G [01:44<01:04, 54.9MB/s]
Downloading best_ckpt_mp0.tar: 60%|██████ | 5.02G/8.31G [01:44<01:10, 50.1MB/s]
Downloading best_ckpt_mp0.tar: 61%|██████ | 5.03G/8.31G [01:44<01:08, 51.5MB/s]
Downloading best_ckpt_mp0.tar: 61%|██████ | 5.04G/8.31G [01:45<01:27, 40.1MB/s]
Downloading best_ckpt_mp0.tar: 61%|██████ | 5.05G/8.31G [01:45<01:29, 39.0MB/s]
Downloading best_ckpt_mp0.tar: 61%|██████ | 5.06G/8.31G [01:45<01:26, 40.6MB/s]
Downloading best_ckpt_mp0.tar: 61%|██████ | 5.07G/8.31G [01:45<01:33, 37.2MB/s]
Downloading best_ckpt_mp0.tar: 61%|██████ | 5.08G/8.31G [01:46<01:24, 41.3MB/s]
Downloading best_ckpt_mp0.tar: 61%|██████ | 5.09G/8.31G [01:46<01:13, 47.4MB/s]
Downloading best_ckpt_mp0.tar: 61%|██████▏ | 5.10G/8.31G [01:46<01:13, 46.8MB/s]
Downloading best_ckpt_mp0.tar: 61%|██████▏ | 5.11G/8.31G [01:46<01:05, 52.8MB/s]
Downloading best_ckpt_mp0.tar: 62%|██████▏ | 5.12G/8.31G [01:46<01:03, 53.7MB/s]
Downloading best_ckpt_mp0.tar: 62%|██████▏ | 5.13G/8.31G [01:47<01:13, 46.6MB/s]
Downloading best_ckpt_mp0.tar: 62%|██████▏ | 5.14G/8.31G [01:47<01:03, 53.2MB/s]
Downloading best_ckpt_mp0.tar: 62%|██████▏ | 5.15G/8.31G [01:47<01:20, 42.2MB/s]
Downloading best_ckpt_mp0.tar: 62%|██████▏ | 5.16G/8.31G [01:47<01:07, 50.3MB/s]
Downloading best_ckpt_mp0.tar: 62%|██████▏ | 5.17G/8.31G [01:47<00:57, 58.3MB/s]
Downloading best_ckpt_mp0.tar: 62%|██████▏ | 5.18G/8.31G [01:47<00:56, 59.9MB/s]
Downloading best_ckpt_mp0.tar: 62%|██████▏ | 5.19G/8.31G [01:48<01:13, 45.6MB/s]
Downloading best_ckpt_mp0.tar: 63%|██████▎ | 5.20G/8.31G [01:48<01:06, 50.4MB/s]
Downloading best_ckpt_mp0.tar: 63%|██████▎ | 5.21G/8.31G [01:48<01:01, 54.3MB/s]
Downloading best_ckpt_mp0.tar: 63%|██████▎ | 5.21G/8.31G [01:48<00:58, 56.7MB/s]
Downloading best_ckpt_mp0.tar: 63%|██████▎ | 5.22G/8.31G [01:48<00:54, 60.9MB/s]
Downloading best_ckpt_mp0.tar: 63%|██████▎ | 5.23G/8.31G [01:49<00:48, 67.7MB/s]
Downloading best_ckpt_mp0.tar: 63%|██████▎ | 5.24G/8.31G [01:49<00:51, 63.5MB/s]
Downloading best_ckpt_mp0.tar: 63%|██████▎ | 5.25G/8.31G [01:49<01:00, 53.8MB/s]
Downloading best_ckpt_mp0.tar: 63%|██████▎ | 5.26G/8.31G [01:49<00:54, 59.7MB/s]
Downloading best_ckpt_mp0.tar: 63%|██████▎ | 5.27G/8.31G [01:49<00:53, 61.4MB/s]
Downloading best_ckpt_mp0.tar: 64%|██████▎ | 5.28G/8.31G [01:50<01:03, 51.4MB/s]
Downloading best_ckpt_mp0.tar: 64%|██████▎ | 5.29G/8.31G [01:50<00:57, 56.3MB/s]
Downloading best_ckpt_mp0.tar: 64%|██████▍ | 5.30G/8.31G [01:50<01:01, 52.6MB/s]
Downloading best_ckpt_mp0.tar: 64%|██████▍ | 5.31G/8.31G [01:50<01:02, 51.4MB/s]
Downloading best_ckpt_mp0.tar: 64%|██████▍ | 5.32G/8.31G [01:50<01:06, 48.2MB/s]
Downloading best_ckpt_mp0.tar: 64%|██████▍ | 5.33G/8.31G [01:51<01:02, 51.1MB/s]
Downloading best_ckpt_mp0.tar: 64%|██████▍ | 5.34G/8.31G [01:51<01:00, 52.6MB/s]
Downloading best_ckpt_mp0.tar: 64%|██████▍ | 5.35G/8.31G [01:51<01:13, 42.9MB/s]
Downloading best_ckpt_mp0.tar: 65%|██████▍ | 5.36G/8.31G [01:51<01:18, 40.1MB/s]
Downloading best_ckpt_mp0.tar: 65%|██████▍ | 5.37G/8.31G [01:52<01:19, 39.6MB/s]
Downloading best_ckpt_mp0.tar: 65%|██████▍ | 5.38G/8.31G [01:52<01:09, 45.0MB/s]
Downloading best_ckpt_mp0.tar: 65%|██████▍ | 5.39G/8.31G [01:52<01:04, 48.5MB/s]
Downloading best_ckpt_mp0.tar: 65%|██████▍ | 5.40G/8.31G [01:52<01:04, 48.1MB/s]
Downloading best_ckpt_mp0.tar: 65%|██████▌ | 5.41G/8.31G [01:52<00:59, 52.6MB/s]
Downloading best_ckpt_mp0.tar: 65%|██████▌ | 5.42G/8.31G [01:53<00:56, 54.7MB/s]
Downloading best_ckpt_mp0.tar: 65%|██████▌ | 5.43G/8.31G [01:53<00:52, 58.7MB/s]
Downloading best_ckpt_mp0.tar: 65%|██████▌ | 5.44G/8.31G [01:53<01:02, 49.7MB/s]
Downloading best_ckpt_mp0.tar: 66%|██████▌ | 5.45G/8.31G [01:53<00:53, 57.1MB/s]
Downloading best_ckpt_mp0.tar: 66%|██████▌ | 5.46G/8.31G [01:53<01:01, 49.9MB/s]
Downloading best_ckpt_mp0.tar: 66%|██████▌ | 5.47G/8.31G [01:54<00:58, 52.1MB/s]
Downloading best_ckpt_mp0.tar: 66%|██████▌ | 5.48G/8.31G [01:54<00:52, 57.4MB/s]
Downloading best_ckpt_mp0.tar: 66%|██████▌ | 5.49G/8.31G [01:54<00:49, 61.6MB/s]
Downloading best_ckpt_mp0.tar: 66%|██████▌ | 5.50G/8.31G [01:54<00:52, 57.3MB/s]
Downloading best_ckpt_mp0.tar: 66%|██████▋ | 5.51G/8.31G [01:54<01:12, 41.7MB/s]
Downloading best_ckpt_mp0.tar: 66%|██████▋ | 5.52G/8.31G [01:55<01:01, 48.4MB/s]
Downloading best_ckpt_mp0.tar: 67%|██████▋ | 5.53G/8.31G [01:55<00:54, 54.8MB/s]
Downloading best_ckpt_mp0.tar: 67%|██████▋ | 5.54G/8.31G [01:55<01:11, 41.9MB/s]
Downloading best_ckpt_mp0.tar: 67%|██████▋ | 5.55G/8.31G [01:55<00:59, 49.9MB/s]
Downloading best_ckpt_mp0.tar: 67%|██████▋ | 5.56G/8.31G [01:55<00:56, 52.3MB/s]
Downloading best_ckpt_mp0.tar: 67%|██████▋ | 5.57G/8.31G [01:56<01:02, 47.0MB/s]
Downloading best_ckpt_mp0.tar: 67%|██████▋ | 5.58G/8.31G [01:56<00:56, 51.8MB/s]
Downloading best_ckpt_mp0.tar: 67%|██████▋ | 5.59G/8.31G [01:56<00:49, 59.4MB/s]
Downloading best_ckpt_mp0.tar: 67%|██████▋ | 5.60G/8.31G [01:56<00:48, 60.4MB/s]
Downloading best_ckpt_mp0.tar: 67%|██████▋ | 5.61G/8.31G [01:56<00:45, 63.9MB/s]
Downloading best_ckpt_mp0.tar: 68%|██████▊ | 5.62G/8.31G [01:56<00:46, 62.8MB/s]
Downloading best_ckpt_mp0.tar: 68%|██████▊ | 5.62G/8.31G [01:57<00:53, 53.7MB/s]
Downloading best_ckpt_mp0.tar: 68%|██████▊ | 5.63G/8.31G [01:57<00:49, 57.8MB/s]
Downloading best_ckpt_mp0.tar: 68%|██████▊ | 5.64G/8.31G [01:57<00:48, 58.6MB/s]
Downloading best_ckpt_mp0.tar: 68%|██████▊ | 5.65G/8.31G [01:57<00:46, 61.1MB/s]
Downloading best_ckpt_mp0.tar: 68%|██████▊ | 5.66G/8.31G [01:57<00:43, 64.7MB/s]
Downloading best_ckpt_mp0.tar: 68%|██████▊ | 5.67G/8.31G [01:57<00:40, 70.1MB/s]
Downloading best_ckpt_mp0.tar: 68%|██████▊ | 5.68G/8.31G [01:58<00:56, 49.6MB/s]
Downloading best_ckpt_mp0.tar: 69%|██████▊ | 5.69G/8.31G [01:58<00:51, 54.5MB/s]
Downloading best_ckpt_mp0.tar: 69%|██████▊ | 5.70G/8.31G [01:58<00:52, 53.3MB/s]
Downloading best_ckpt_mp0.tar: 69%|██████▉ | 5.71G/8.31G [01:58<00:48, 57.8MB/s]
Downloading best_ckpt_mp0.tar: 69%|██████▉ | 5.72G/8.31G [01:58<00:46, 59.2MB/s]
Downloading best_ckpt_mp0.tar: 69%|██████▉ | 5.73G/8.31G [01:59<00:44, 62.1MB/s]
Downloading best_ckpt_mp0.tar: 69%|██████▉ | 5.74G/8.31G [01:59<00:42, 64.8MB/s]
Downloading best_ckpt_mp0.tar: 69%|██████▉ | 5.75G/8.31G [01:59<00:50, 54.0MB/s]
Downloading best_ckpt_mp0.tar: 69%|██████▉ | 5.76G/8.31G [01:59<00:48, 56.3MB/s]
Downloading best_ckpt_mp0.tar: 69%|██████▉ | 5.77G/8.31G [01:59<00:44, 61.1MB/s]
Downloading best_ckpt_mp0.tar: 70%|██████▉ | 5.78G/8.31G [02:00<00:42, 63.6MB/s]
Downloading best_ckpt_mp0.tar: 70%|██████▉ | 5.79G/8.31G [02:00<00:44, 60.5MB/s]
Downloading best_ckpt_mp0.tar: 70%|██████▉ | 5.80G/8.31G [02:00<01:03, 42.3MB/s]
Downloading best_ckpt_mp0.tar: 70%|██████▉ | 5.81G/8.31G [02:00<00:53, 50.3MB/s]
Downloading best_ckpt_mp0.tar: 70%|███████ | 5.82G/8.31G [02:00<00:46, 58.1MB/s]
Downloading best_ckpt_mp0.tar: 70%|███████ | 5.83G/8.31G [02:01<00:45, 58.5MB/s]
Downloading best_ckpt_mp0.tar: 70%|███████ | 5.84G/8.31G [02:01<00:47, 55.9MB/s]
Downloading best_ckpt_mp0.tar: 70%|███████ | 5.85G/8.31G [02:01<00:46, 56.5MB/s]
Downloading best_ckpt_mp0.tar: 71%|███████ | 5.86G/8.31G [02:01<00:45, 58.0MB/s]
Downloading best_ckpt_mp0.tar: 71%|███████ | 5.87G/8.31G [02:01<00:45, 58.2MB/s]
Downloading best_ckpt_mp0.tar: 71%|███████ | 5.88G/8.31G [02:02<01:04, 40.8MB/s]
Downloading best_ckpt_mp0.tar: 71%|███████ | 5.89G/8.31G [02:02<01:00, 43.0MB/s]
Downloading best_ckpt_mp0.tar: 71%|███████ | 5.90G/8.31G [02:02<00:56, 45.7MB/s]
Downloading best_ckpt_mp0.tar: 71%|███████ | 5.91G/8.31G [02:02<00:58, 43.8MB/s]
Downloading best_ckpt_mp0.tar: 71%|███████ | 5.92G/8.31G [02:03<00:51, 49.9MB/s]
Downloading best_ckpt_mp0.tar: 71%|███████▏ | 5.93G/8.31G [02:03<00:48, 52.7MB/s]
Downloading best_ckpt_mp0.tar: 71%|███████▏ | 5.94G/8.31G [02:03<00:53, 47.7MB/s]
Downloading best_ckpt_mp0.tar: 72%|███████▏ | 5.95G/8.31G [02:03<00:48, 52.8MB/s]
Downloading best_ckpt_mp0.tar: 72%|███████▏ | 5.96G/8.31G [02:03<00:42, 59.7MB/s]
Downloading best_ckpt_mp0.tar: 72%|███████▏ | 5.97G/8.31G [02:03<00:40, 62.1MB/s]
Downloading best_ckpt_mp0.tar: 72%|███████▏ | 5.98G/8.31G [02:04<00:41, 61.1MB/s]
Downloading best_ckpt_mp0.tar: 72%|███████▏ | 5.99G/8.31G [02:04<00:39, 62.6MB/s]
Downloading best_ckpt_mp0.tar: 72%|███████▏ | 6.00G/8.31G [02:04<00:58, 42.2MB/s]
Downloading best_ckpt_mp0.tar: 72%|███████▏ | 6.01G/8.31G [02:04<00:55, 44.2MB/s]
Downloading best_ckpt_mp0.tar: 72%|███████▏ | 6.02G/8.31G [02:05<00:51, 47.5MB/s]
Downloading best_ckpt_mp0.tar: 73%|███████▎ | 6.03G/8.31G [02:05<00:47, 51.1MB/s]
Downloading best_ckpt_mp0.tar: 73%|███████▎ | 6.04G/8.31G [02:05<00:45, 54.1MB/s]
Downloading best_ckpt_mp0.tar: 73%|███████▎ | 6.04G/8.31G [02:05<00:43, 56.5MB/s]
Downloading best_ckpt_mp0.tar: 73%|███████▎ | 6.05G/8.31G [02:05<00:40, 59.6MB/s]
Downloading best_ckpt_mp0.tar: 73%|███████▎ | 6.06G/8.31G [02:05<00:44, 53.7MB/s]
Downloading best_ckpt_mp0.tar: 73%|███████▎ | 6.07G/8.31G [02:06<00:39, 60.1MB/s]
Downloading best_ckpt_mp0.tar: 73%|███████▎ | 6.08G/8.31G [02:06<00:35, 67.3MB/s]
Downloading best_ckpt_mp0.tar: 73%|███████▎ | 6.09G/8.31G [02:06<00:38, 61.1MB/s]
Downloading best_ckpt_mp0.tar: 73%|███████▎ | 6.10G/8.31G [02:06<00:35, 67.1MB/s]
Downloading best_ckpt_mp0.tar: 74%|███████▎ | 6.11G/8.31G [02:06<00:32, 72.2MB/s]
Downloading best_ckpt_mp0.tar: 74%|███████▎ | 6.12G/8.31G [02:06<00:41, 57.1MB/s]
Downloading best_ckpt_mp0.tar: 74%|███████▍ | 6.13G/8.31G [02:07<00:36, 64.5MB/s]
Downloading best_ckpt_mp0.tar: 74%|███████▍ | 6.14G/8.31G [02:07<00:38, 60.6MB/s]
Downloading best_ckpt_mp0.tar: 74%|███████▍ | 6.15G/8.31G [02:07<00:36, 63.9MB/s]
Downloading best_ckpt_mp0.tar: 74%|███████▍ | 6.16G/8.31G [02:07<00:35, 64.8MB/s]
Downloading best_ckpt_mp0.tar: 74%|███████▍ | 6.17G/8.31G [02:07<00:34, 66.9MB/s]
Downloading best_ckpt_mp0.tar: 74%|███████▍ | 6.18G/8.31G [02:07<00:34, 66.3MB/s]
Downloading best_ckpt_mp0.tar: 75%|███████▍ | 6.19G/8.31G [02:08<00:40, 55.9MB/s]
Downloading best_ckpt_mp0.tar: 75%|███████▍ | 6.20G/8.31G [02:08<00:35, 63.0MB/s]
Downloading best_ckpt_mp0.tar: 75%|███████▍ | 6.21G/8.31G [02:08<00:34, 65.1MB/s]
Downloading best_ckpt_mp0.tar: 75%|███████▍ | 6.22G/8.31G [02:08<00:36, 61.5MB/s]
Downloading best_ckpt_mp0.tar: 75%|███████▍ | 6.23G/8.31G [02:08<00:38, 57.7MB/s]
Downloading best_ckpt_mp0.tar: 75%|███████▌ | 6.24G/8.31G [02:08<00:36, 60.4MB/s]
Downloading best_ckpt_mp0.tar: 75%|███████▌ | 6.25G/8.31G [02:09<00:42, 51.8MB/s]
Downloading best_ckpt_mp0.tar: 75%|███████▌ | 6.26G/8.31G [02:09<00:39, 55.3MB/s]
Downloading best_ckpt_mp0.tar: 75%|███████▌ | 6.27G/8.31G [02:09<00:36, 59.3MB/s]
Downloading best_ckpt_mp0.tar: 76%|███████▌ | 6.28G/8.31G [02:09<00:35, 62.2MB/s]
Downloading best_ckpt_mp0.tar: 76%|███████▌ | 6.29G/8.31G [02:09<00:35, 61.6MB/s]
Downloading best_ckpt_mp0.tar: 76%|███████▌ | 6.30G/8.31G [02:09<00:34, 62.1MB/s]
Downloading best_ckpt_mp0.tar: 76%|███████▌ | 6.31G/8.31G [02:10<00:41, 51.4MB/s]
Downloading best_ckpt_mp0.tar: 76%|███████▌ | 6.32G/8.31G [02:10<00:41, 51.1MB/s]
Downloading best_ckpt_mp0.tar: 76%|███████▌ | 6.33G/8.31G [02:10<00:39, 53.4MB/s]
Downloading best_ckpt_mp0.tar: 76%|███████▋ | 6.34G/8.31G [02:10<00:39, 54.2MB/s]
Downloading best_ckpt_mp0.tar: 76%|███████▋ | 6.35G/8.31G [02:11<00:39, 53.7MB/s]
Downloading best_ckpt_mp0.tar: 77%|███████▋ | 6.36G/8.31G [02:11<00:37, 56.4MB/s]
Downloading best_ckpt_mp0.tar: 77%|███████▋ | 6.37G/8.31G [02:11<00:35, 58.5MB/s]
Downloading best_ckpt_mp0.tar: 77%|███████▋ | 6.38G/8.31G [02:11<00:41, 50.1MB/s]
Downloading best_ckpt_mp0.tar: 77%|███████▋ | 6.39G/8.31G [02:11<00:38, 53.5MB/s]
Downloading best_ckpt_mp0.tar: 77%|███████▋ | 6.40G/8.31G [02:11<00:36, 56.2MB/s]
Downloading best_ckpt_mp0.tar: 77%|███████▋ | 6.41G/8.31G [02:12<00:34, 59.2MB/s]
Downloading best_ckpt_mp0.tar: 77%|███████▋ | 6.42G/8.31G [02:12<00:32, 62.9MB/s]
Downloading best_ckpt_mp0.tar: 77%|███████▋ | 6.43G/8.31G [02:12<00:33, 60.1MB/s]
Downloading best_ckpt_mp0.tar: 77%|███████▋ | 6.44G/8.31G [02:12<00:39, 50.9MB/s]
Downloading best_ckpt_mp0.tar: 78%|███████▊ | 6.45G/8.31G [02:12<00:37, 53.6MB/s]
Downloading best_ckpt_mp0.tar: 78%|███████▊ | 6.46G/8.31G [02:13<00:37, 53.6MB/s]
Downloading best_ckpt_mp0.tar: 78%|███████▊ | 6.46G/8.31G [02:13<00:36, 54.2MB/s]
Downloading best_ckpt_mp0.tar: 78%|███████▊ | 6.47G/8.31G [02:13<00:38, 51.4MB/s]
Downloading best_ckpt_mp0.tar: 78%|███████▊ | 6.48G/8.31G [02:13<00:35, 54.8MB/s]
Downloading best_ckpt_mp0.tar: 78%|███████▊ | 6.49G/8.31G [02:13<00:36, 53.9MB/s]
Downloading best_ckpt_mp0.tar: 78%|███████▊ | 6.50G/8.31G [02:14<00:42, 45.4MB/s]
Downloading best_ckpt_mp0.tar: 78%|███████▊ | 6.51G/8.31G [02:14<00:45, 42.8MB/s]
Downloading best_ckpt_mp0.tar: 79%|███████▊ | 6.52G/8.31G [02:14<00:47, 40.5MB/s]
Downloading best_ckpt_mp0.tar: 79%|███████▊ | 6.53G/8.31G [02:15<00:49, 38.5MB/s]
Downloading best_ckpt_mp0.tar: 79%|███████▊ | 6.54G/8.31G [02:15<00:46, 40.5MB/s]
Downloading best_ckpt_mp0.tar: 79%|███████▉ | 6.55G/8.31G [02:15<00:46, 40.5MB/s]
Downloading best_ckpt_mp0.tar: 79%|███████▉ | 6.56G/8.31G [02:15<00:51, 36.3MB/s]
Downloading best_ckpt_mp0.tar: 79%|███████▉ | 6.57G/8.31G [02:16<00:47, 39.0MB/s]
Downloading best_ckpt_mp0.tar: 79%|███████▉ | 6.58G/8.31G [02:16<00:44, 41.5MB/s]
Downloading best_ckpt_mp0.tar: 79%|███████▉ | 6.59G/8.31G [02:16<00:44, 41.2MB/s]
Downloading best_ckpt_mp0.tar: 79%|███████▉ | 6.60G/8.31G [02:16<00:41, 44.4MB/s]
Downloading best_ckpt_mp0.tar: 80%|███████▉ | 6.61G/8.31G [02:17<00:42, 42.6MB/s]
Downloading best_ckpt_mp0.tar: 80%|███████▉ | 6.62G/8.31G [02:17<00:49, 36.5MB/s]
Downloading best_ckpt_mp0.tar: 80%|███████▉ | 6.63G/8.31G [02:17<00:46, 38.9MB/s]
Downloading best_ckpt_mp0.tar: 80%|███████▉ | 6.64G/8.31G [02:17<00:44, 40.6MB/s]
Downloading best_ckpt_mp0.tar: 80%|████████ | 6.65G/8.31G [02:18<00:39, 45.6MB/s]
Downloading best_ckpt_mp0.tar: 80%|████████ | 6.66G/8.31G [02:18<00:38, 45.8MB/s]
Downloading best_ckpt_mp0.tar: 80%|████████ | 6.67G/8.31G [02:18<00:41, 42.7MB/s]
Downloading best_ckpt_mp0.tar: 80%|████████ | 6.68G/8.31G [02:18<00:40, 43.2MB/s]
Downloading best_ckpt_mp0.tar: 81%|████████ | 6.69G/8.31G [02:19<00:45, 38.2MB/s]
Downloading best_ckpt_mp0.tar: 81%|████████ | 6.70G/8.31G [02:19<00:41, 41.7MB/s]
Downloading best_ckpt_mp0.tar: 81%|████████ | 6.71G/8.31G [02:19<00:40, 42.9MB/s]
Downloading best_ckpt_mp0.tar: 81%|████████ | 6.72G/8.31G [02:19<00:40, 42.6MB/s]
Downloading best_ckpt_mp0.tar: 81%|████████ | 6.73G/8.31G [02:20<00:36, 46.2MB/s]
Downloading best_ckpt_mp0.tar: 81%|████████ | 6.74G/8.31G [02:20<00:34, 49.2MB/s]
Downloading best_ckpt_mp0.tar: 81%|████████ | 6.75G/8.31G [02:20<00:38, 43.1MB/s]
Downloading best_ckpt_mp0.tar: 81%|████████▏ | 6.76G/8.31G [02:20<00:36, 46.1MB/s]
Downloading best_ckpt_mp0.tar: 81%|████████▏ | 6.77G/8.31G [02:20<00:33, 50.0MB/s]
Downloading best_ckpt_mp0.tar: 82%|████████▏ | 6.78G/8.31G [02:21<00:29, 55.0MB/s]
Downloading best_ckpt_mp0.tar: 82%|████████▏ | 6.79G/8.31G [02:21<00:27, 58.8MB/s]
Downloading best_ckpt_mp0.tar: 82%|████████▏ | 6.80G/8.31G [02:21<00:26, 61.7MB/s]
Downloading best_ckpt_mp0.tar: 82%|████████▏ | 6.81G/8.31G [02:21<00:26, 60.0MB/s]
Downloading best_ckpt_mp0.tar: 82%|████████▏ | 6.82G/8.31G [02:21<00:34, 46.7MB/s]
Downloading best_ckpt_mp0.tar: 82%|████████▏ | 6.83G/8.31G [02:22<00:34, 46.7MB/s]
Downloading best_ckpt_mp0.tar: 82%|████████▏ | 6.84G/8.31G [02:22<00:33, 47.1MB/s]
Downloading best_ckpt_mp0.tar: 82%|████████▏ | 6.85G/8.31G [02:22<00:34, 45.2MB/s]
Downloading best_ckpt_mp0.tar: 83%|████████▎ | 6.86G/8.31G [02:22<00:32, 47.4MB/s]
Downloading best_ckpt_mp0.tar: 83%|████████▎ | 6.87G/8.31G [02:23<00:38, 40.4MB/s]
Downloading best_ckpt_mp0.tar: 83%|████████▎ | 6.88G/8.31G [02:23<00:39, 39.4MB/s]
Downloading best_ckpt_mp0.tar: 83%|████████▎ | 6.88G/8.31G [02:23<00:34, 44.2MB/s]
Downloading best_ckpt_mp0.tar: 83%|████████▎ | 6.89G/8.31G [02:23<00:30, 49.9MB/s]
Downloading best_ckpt_mp0.tar: 83%|████████▎ | 6.90G/8.31G [02:23<00:29, 52.0MB/s]
Downloading best_ckpt_mp0.tar: 83%|████████▎ | 6.91G/8.31G [02:24<00:28, 51.9MB/s]
Downloading best_ckpt_mp0.tar: 83%|████████▎ | 6.92G/8.31G [02:24<00:27, 53.5MB/s]
Downloading best_ckpt_mp0.tar: 83%|████████▎ | 6.93G/8.31G [02:24<00:32, 45.6MB/s]
Downloading best_ckpt_mp0.tar: 84%|████████▎ | 6.94G/8.31G [02:24<00:30, 48.2MB/s]
Downloading best_ckpt_mp0.tar: 84%|████████▎ | 6.95G/8.31G [02:24<00:27, 52.3MB/s]
Downloading best_ckpt_mp0.tar: 84%|████████▍ | 6.96G/8.31G [02:25<00:28, 50.7MB/s]
Downloading best_ckpt_mp0.tar: 84%|████████▍ | 6.97G/8.31G [02:25<00:27, 52.7MB/s]
Downloading best_ckpt_mp0.tar: 84%|████████▍ | 6.98G/8.31G [02:25<00:25, 55.9MB/s]
Downloading best_ckpt_mp0.tar: 84%|████████▍ | 6.99G/8.31G [02:25<00:24, 58.1MB/s]
Downloading best_ckpt_mp0.tar: 84%|████████▍ | 7.00G/8.31G [02:25<00:27, 50.2MB/s]
Downloading best_ckpt_mp0.tar: 84%|████████▍ | 7.01G/8.31G [02:26<00:26, 52.2MB/s]
Downloading best_ckpt_mp0.tar: 85%|████████▍ | 7.02G/8.31G [02:26<00:24, 56.2MB/s]
Downloading best_ckpt_mp0.tar: 85%|████████▍ | 7.03G/8.31G [02:26<00:22, 61.0MB/s]
Downloading best_ckpt_mp0.tar: 85%|████████▍ | 7.04G/8.31G [02:26<00:22, 61.2MB/s]
Downloading best_ckpt_mp0.tar: 85%|████████▍ | 7.05G/8.31G [02:26<00:21, 62.8MB/s]
Downloading best_ckpt_mp0.tar: 85%|████████▍ | 7.06G/8.31G [02:27<00:26, 51.0MB/s]
Downloading best_ckpt_mp0.tar: 85%|████████▌ | 7.07G/8.31G [02:27<00:24, 55.2MB/s]
Downloading best_ckpt_mp0.tar: 85%|████████▌ | 7.08G/8.31G [02:27<00:21, 62.8MB/s]
Downloading best_ckpt_mp0.tar: 85%|████████▌ | 7.09G/8.31G [02:27<00:20, 63.9MB/s]
Downloading best_ckpt_mp0.tar: 85%|████████▌ | 7.10G/8.31G [02:27<00:20, 63.7MB/s]
Downloading best_ckpt_mp0.tar: 86%|████████▌ | 7.11G/8.31G [02:27<00:19, 66.6MB/s]
Downloading best_ckpt_mp0.tar: 86%|████████▌ | 7.12G/8.31G [02:27<00:19, 66.3MB/s]
Downloading best_ckpt_mp0.tar: 86%|████████▌ | 7.13G/8.31G [02:28<00:24, 52.4MB/s]
Downloading best_ckpt_mp0.tar: 86%|████████▌ | 7.14G/8.31G [02:28<00:22, 55.2MB/s]
Downloading best_ckpt_mp0.tar: 86%|████████▌ | 7.15G/8.31G [02:28<00:20, 59.5MB/s]
Downloading best_ckpt_mp0.tar: 86%|████████▌ | 7.16G/8.31G [02:28<00:19, 63.1MB/s]
Downloading best_ckpt_mp0.tar: 86%|████████▋ | 7.17G/8.31G [02:28<00:19, 61.4MB/s]
Downloading best_ckpt_mp0.tar: 86%|████████▋ | 7.18G/8.31G [02:29<00:20, 60.2MB/s]
Downloading best_ckpt_mp0.tar: 87%|████████▋ | 7.19G/8.31G [02:29<00:23, 50.2MB/s]
Downloading best_ckpt_mp0.tar: 87%|████████▋ | 7.20G/8.31G [02:29<00:23, 50.8MB/s]
Downloading best_ckpt_mp0.tar: 87%|████████▋ | 7.21G/8.31G [02:29<00:23, 51.3MB/s]
Downloading best_ckpt_mp0.tar: 87%|████████▋ | 7.22G/8.31G [02:29<00:21, 53.9MB/s]
Downloading best_ckpt_mp0.tar: 87%|████████▋ | 7.23G/8.31G [02:30<00:20, 57.1MB/s]
Downloading best_ckpt_mp0.tar: 87%|████████▋ | 7.24G/8.31G [02:30<00:19, 59.1MB/s]
Downloading best_ckpt_mp0.tar: 87%|████████▋ | 7.25G/8.31G [02:30<00:23, 48.9MB/s]
Downloading best_ckpt_mp0.tar: 87%|████████▋ | 7.26G/8.31G [02:30<00:21, 52.8MB/s]
Downloading best_ckpt_mp0.tar: 87%|████████▋ | 7.27G/8.31G [02:30<00:20, 55.6MB/s]
Downloading best_ckpt_mp0.tar: 88%|████████▊ | 7.28G/8.31G [02:31<00:19, 57.8MB/s]
Downloading best_ckpt_mp0.tar: 88%|████████▊ | 7.29G/8.31G [02:31<00:20, 53.9MB/s]
Downloading best_ckpt_mp0.tar: 88%|████████▊ | 7.29G/8.31G [02:31<00:19, 56.9MB/s]
Downloading best_ckpt_mp0.tar: 88%|████████▊ | 7.30G/8.31G [02:31<00:18, 58.6MB/s]
Downloading best_ckpt_mp0.tar: 88%|████████▊ | 7.31G/8.31G [02:31<00:20, 52.1MB/s]
Downloading best_ckpt_mp0.tar: 88%|████████▊ | 7.32G/8.31G [02:31<00:18, 56.3MB/s]
Downloading best_ckpt_mp0.tar: 88%|████████▊ | 7.33G/8.31G [02:32<00:18, 57.3MB/s]
Downloading best_ckpt_mp0.tar: 88%|████████▊ | 7.34G/8.31G [02:32<00:17, 58.3MB/s]
Downloading best_ckpt_mp0.tar: 88%|████████▊ | 7.35G/8.31G [02:32<00:17, 59.3MB/s]
Downloading best_ckpt_mp0.tar: 89%|████████▊ | 7.36G/8.31G [02:32<00:16, 61.8MB/s]
Downloading best_ckpt_mp0.tar: 89%|████████▊ | 7.37G/8.31G [02:32<00:19, 51.9MB/s]
Downloading best_ckpt_mp0.tar: 89%|████████▉ | 7.38G/8.31G [02:33<00:18, 54.0MB/s]
Downloading best_ckpt_mp0.tar: 89%|████████▉ | 7.39G/8.31G [02:33<00:16, 58.8MB/s]
Downloading best_ckpt_mp0.tar: 89%|████████▉ | 7.40G/8.31G [02:33<00:14, 65.3MB/s]
Downloading best_ckpt_mp0.tar: 89%|████████▉ | 7.41G/8.31G [02:33<00:14, 68.5MB/s]
Downloading best_ckpt_mp0.tar: 89%|████████▉ | 7.42G/8.31G [02:33<00:13, 70.2MB/s]
Downloading best_ckpt_mp0.tar: 89%|████████▉ | 7.43G/8.31G [02:33<00:13, 71.6MB/s]
Downloading best_ckpt_mp0.tar: 90%|████████▉ | 7.44G/8.31G [02:34<00:15, 59.5MB/s]
Downloading best_ckpt_mp0.tar: 90%|████████▉ | 7.45G/8.31G [02:34<00:15, 61.2MB/s]
Downloading best_ckpt_mp0.tar: 90%|████████▉ | 7.46G/8.31G [02:34<00:14, 64.0MB/s]
Downloading best_ckpt_mp0.tar: 90%|████████▉ | 7.47G/8.31G [02:34<00:14, 64.0MB/s]
Downloading best_ckpt_mp0.tar: 90%|█████████ | 7.48G/8.31G [02:34<00:13, 67.3MB/s]
Downloading best_ckpt_mp0.tar: 90%|█████████ | 7.49G/8.31G [02:34<00:12, 69.3MB/s]
Downloading best_ckpt_mp0.tar: 90%|█████████ | 7.50G/8.31G [02:34<00:14, 60.2MB/s]
Downloading best_ckpt_mp0.tar: 90%|█████████ | 7.51G/8.31G [02:35<00:14, 60.4MB/s]
Downloading best_ckpt_mp0.tar: 90%|█████████ | 7.52G/8.31G [02:35<00:14, 59.6MB/s]
Downloading best_ckpt_mp0.tar: 91%|█████████ | 7.53G/8.31G [02:35<00:14, 58.5MB/s]
Downloading best_ckpt_mp0.tar: 91%|█████████ | 7.54G/8.31G [02:35<00:14, 58.2MB/s]
Downloading best_ckpt_mp0.tar: 91%|█████████ | 7.55G/8.31G [02:35<00:13, 61.6MB/s]
Downloading best_ckpt_mp0.tar: 91%|█████████ | 7.56G/8.31G [02:36<00:15, 53.1MB/s]
Downloading best_ckpt_mp0.tar: 91%|█████████ | 7.57G/8.31G [02:36<00:13, 57.6MB/s]
Downloading best_ckpt_mp0.tar: 91%|█████████ | 7.58G/8.31G [02:36<00:12, 60.6MB/s]
Downloading best_ckpt_mp0.tar: 91%|█████████▏| 7.59G/8.31G [02:36<00:12, 61.8MB/s]
Downloading best_ckpt_mp0.tar: 91%|█████████▏| 7.60G/8.31G [02:36<00:11, 64.2MB/s]
Downloading best_ckpt_mp0.tar: 92%|█████████▏| 7.61G/8.31G [02:36<00:10, 69.2MB/s]
Downloading best_ckpt_mp0.tar: 92%|█████████▏| 7.62G/8.31G [02:36<00:10, 69.8MB/s]
Downloading best_ckpt_mp0.tar: 92%|█████████▏| 7.63G/8.31G [02:37<00:12, 57.2MB/s]
Downloading best_ckpt_mp0.tar: 92%|█████████▏| 7.64G/8.31G [02:37<00:12, 57.1MB/s]
Downloading best_ckpt_mp0.tar: 92%|█████████▏| 7.65G/8.31G [02:37<00:11, 60.2MB/s]
Downloading best_ckpt_mp0.tar: 92%|█████████▏| 7.66G/8.31G [02:37<00:11, 59.9MB/s]
Downloading best_ckpt_mp0.tar: 92%|█████████▏| 7.67G/8.31G [02:37<00:11, 58.5MB/s]
Downloading best_ckpt_mp0.tar: 92%|█████████▏| 7.68G/8.31G [02:38<00:11, 60.1MB/s]
Downloading best_ckpt_mp0.tar: 92%|█████████▏| 7.69G/8.31G [02:38<00:14, 47.5MB/s]
Downloading best_ckpt_mp0.tar: 93%|█████████▎| 7.70G/8.31G [02:38<00:13, 47.8MB/s]
Downloading best_ckpt_mp0.tar: 93%|█████████▎| 7.71G/8.31G [02:38<00:12, 50.6MB/s]
Downloading best_ckpt_mp0.tar: 93%|█████████▎| 7.71G/8.31G [02:39<00:11, 53.7MB/s]
Downloading best_ckpt_mp0.tar: 93%|█████████▎| 7.72G/8.31G [02:39<00:10, 57.5MB/s]
Downloading best_ckpt_mp0.tar: 93%|█████████▎| 7.73G/8.31G [02:39<00:09, 62.2MB/s]
Downloading best_ckpt_mp0.tar: 93%|█████████▎| 7.74G/8.31G [02:39<00:09, 63.0MB/s]
Downloading best_ckpt_mp0.tar: 93%|█████████▎| 7.75G/8.31G [02:39<00:11, 53.0MB/s]
Downloading best_ckpt_mp0.tar: 93%|█████████▎| 7.76G/8.31G [02:39<00:10, 58.2MB/s]
Downloading best_ckpt_mp0.tar: 94%|█████████▎| 7.77G/8.31G [02:40<00:09, 63.7MB/s]
Downloading best_ckpt_mp0.tar: 94%|█████████▎| 7.78G/8.31G [02:40<00:08, 63.7MB/s]
Downloading best_ckpt_mp0.tar: 94%|█████████▍| 7.79G/8.31G [02:40<00:08, 64.2MB/s]
Downloading best_ckpt_mp0.tar: 94%|█████████▍| 7.80G/8.31G [02:40<00:08, 65.5MB/s]
Downloading best_ckpt_mp0.tar: 94%|█████████▍| 7.81G/8.31G [02:40<00:09, 56.3MB/s]
Downloading best_ckpt_mp0.tar: 94%|█████████▍| 7.82G/8.31G [02:40<00:09, 57.6MB/s]
Downloading best_ckpt_mp0.tar: 94%|█████████▍| 7.83G/8.31G [02:41<00:09, 55.0MB/s]
Downloading best_ckpt_mp0.tar: 94%|█████████▍| 7.84G/8.31G [02:41<00:08, 59.1MB/s]
Downloading best_ckpt_mp0.tar: 94%|█████████▍| 7.85G/8.31G [02:41<00:08, 58.9MB/s]
Downloading best_ckpt_mp0.tar: 95%|█████████▍| 7.86G/8.31G [02:41<00:08, 57.0MB/s]
Downloading best_ckpt_mp0.tar: 95%|█████████▍| 7.87G/8.31G [02:41<00:09, 48.6MB/s]
Downloading best_ckpt_mp0.tar: 95%|█████████▍| 7.88G/8.31G [02:42<00:08, 54.1MB/s]
Downloading best_ckpt_mp0.tar: 95%|█████████▍| 7.89G/8.31G [02:42<00:07, 56.5MB/s]
Downloading best_ckpt_mp0.tar: 95%|█████████▌| 7.90G/8.31G [02:42<00:07, 59.7MB/s]
Downloading best_ckpt_mp0.tar: 95%|█████████▌| 7.91G/8.31G [02:42<00:06, 61.7MB/s]
Downloading best_ckpt_mp0.tar: 95%|█████████▌| 7.92G/8.31G [02:42<00:06, 64.0MB/s]
Downloading best_ckpt_mp0.tar: 95%|█████████▌| 7.93G/8.31G [02:42<00:06, 64.0MB/s]
Downloading best_ckpt_mp0.tar: 96%|█████████▌| 7.94G/8.31G [02:43<00:07, 54.9MB/s]
Downloading best_ckpt_mp0.tar: 96%|█████████▌| 7.95G/8.31G [02:43<00:06, 56.2MB/s]
Downloading best_ckpt_mp0.tar: 96%|█████████▌| 7.96G/8.31G [02:43<00:06, 58.0MB/s]
Downloading best_ckpt_mp0.tar: 96%|█████████▌| 7.97G/8.31G [02:43<00:06, 54.3MB/s]
Downloading best_ckpt_mp0.tar: 96%|█████████▌| 7.98G/8.31G [02:43<00:06, 58.0MB/s]
Downloading best_ckpt_mp0.tar: 96%|█████████▌| 7.99G/8.31G [02:43<00:05, 65.7MB/s]
Downloading best_ckpt_mp0.tar: 96%|█████████▋| 8.00G/8.31G [02:44<00:06, 54.4MB/s]
Downloading best_ckpt_mp0.tar: 96%|█████████▋| 8.01G/8.31G [02:44<00:05, 61.5MB/s]
Downloading best_ckpt_mp0.tar: 96%|█████████▋| 8.02G/8.31G [02:44<00:04, 62.7MB/s]
Downloading best_ckpt_mp0.tar: 97%|█████████▋| 8.03G/8.31G [02:44<00:04, 61.9MB/s]
Downloading best_ckpt_mp0.tar: 97%|█████████▋| 8.04G/8.31G [02:44<00:04, 59.9MB/s]
Downloading best_ckpt_mp0.tar: 97%|█████████▋| 8.05G/8.31G [02:45<00:04, 58.5MB/s]
Downloading best_ckpt_mp0.tar: 97%|█████████▋| 8.06G/8.31G [02:45<00:04, 57.6MB/s]
Downloading best_ckpt_mp0.tar: 97%|█████████▋| 8.07G/8.31G [02:45<00:05, 49.4MB/s]
Downloading best_ckpt_mp0.tar: 97%|█████████▋| 8.08G/8.31G [02:45<00:04, 51.2MB/s]
Downloading best_ckpt_mp0.tar: 97%|█████████▋| 8.09G/8.31G [02:45<00:04, 54.0MB/s]
Downloading best_ckpt_mp0.tar: 97%|█████████▋| 8.10G/8.31G [02:46<00:04, 49.7MB/s]
Downloading best_ckpt_mp0.tar: 98%|█████████▊| 8.11G/8.31G [02:46<00:04, 49.5MB/s]
Downloading best_ckpt_mp0.tar: 98%|█████████▊| 8.12G/8.31G [02:46<00:04, 50.0MB/s]
Downloading best_ckpt_mp0.tar: 98%|█████████▊| 8.12G/8.31G [02:46<00:04, 42.0MB/s]
Downloading best_ckpt_mp0.tar: 98%|█████████▊| 8.13G/8.31G [02:47<00:04, 46.2MB/s]
Downloading best_ckpt_mp0.tar: 98%|█████████▊| 8.14G/8.31G [02:47<00:03, 49.8MB/s]
Downloading best_ckpt_mp0.tar: 98%|█████████▊| 8.15G/8.31G [02:47<00:03, 51.0MB/s]
Downloading best_ckpt_mp0.tar: 98%|█████████▊| 8.16G/8.31G [02:47<00:02, 54.5MB/s]
Downloading best_ckpt_mp0.tar: 98%|█████████▊| 8.17G/8.31G [02:47<00:02, 53.2MB/s]
Downloading best_ckpt_mp0.tar: 98%|█████████▊| 8.18G/8.31G [02:48<00:02, 46.2MB/s]
Downloading best_ckpt_mp0.tar: 99%|█████████▊| 8.19G/8.31G [02:48<00:02, 50.3MB/s]
Downloading best_ckpt_mp0.tar: 99%|█████████▊| 8.20G/8.31G [02:48<00:02, 50.5MB/s]
Downloading best_ckpt_mp0.tar: 99%|█████████▉| 8.21G/8.31G [02:48<00:01, 52.5MB/s]
Downloading best_ckpt_mp0.tar: 99%|█████████▉| 8.22G/8.31G [02:48<00:01, 58.3MB/s]
Downloading best_ckpt_mp0.tar: 99%|█████████▉| 8.23G/8.31G [02:48<00:01, 63.1MB/s]
Downloading best_ckpt_mp0.tar: 99%|█████████▉| 8.24G/8.31G [02:49<00:01, 60.6MB/s]
Downloading best_ckpt_mp0.tar: 99%|█████████▉| 8.25G/8.31G [02:49<00:01, 48.2MB/s]
Downloading best_ckpt_mp0.tar: 99%|█████████▉| 8.26G/8.31G [02:49<00:01, 40.9MB/s]
Downloading best_ckpt_mp0.tar: 100%|█████████▉| 8.27G/8.31G [02:49<00:00, 47.1MB/s]
Downloading best_ckpt_mp0.tar: 100%|█████████▉| 8.28G/8.31G [02:50<00:00, 49.4MB/s]
Downloading best_ckpt_mp0.tar: 100%|█████████▉| 8.29G/8.31G [02:50<00:00, 54.5MB/s]
Downloading best_ckpt_mp0.tar: 100%|█████████▉| 8.30G/8.31G [02:50<00:00, 59.2MB/s]
Downloading best_ckpt_mp0.tar: 100%|██████████| 8.31G/8.31G [02:50<00:00, 48.8MB/s]
Downloading best_ckpt_mp0.tar: 100%|██████████| 8.31G/8.31G [02:50<00:00, 52.2MB/s]Run¶
Execute the pipeline. For this example, we use the period of Typhoon Doksuri which has a track over the Taiwan region. en.wikipedia.org/wiki/Typhoon_Doksuri
start_date = datetime(2023, 7, 26, 12)
corrdiff_on_hens_ensemble(
time=[start_date],
nsteps=4,
nensemble=2,
nsamples=3,
prognostic=model,
corrdiff=corrdiff,
data=data,
io=io,
perturbation=perturbation,
batch_size=1,
)
Console output104 lines
Fetching GFS data: 0%| | 0/74 [00:00<?, ?it/s]
Fetching GFS data: 1%|▏ | 1/74 [00:00<00:34, 2.09it/s]
Fetching GFS data: 4%|▍ | 3/74 [00:00<00:12, 5.70it/s]
Fetching GFS data: 8%|▊ | 6/74 [00:00<00:06, 10.62it/s]
Fetching GFS data: 14%|█▎ | 10/74 [00:00<00:03, 17.22it/s]
Fetching GFS data: 18%|█▊ | 13/74 [00:01<00:03, 15.44it/s]
Fetching GFS data: 22%|██▏ | 16/74 [00:01<00:03, 15.04it/s]
Fetching GFS data: 24%|██▍ | 18/74 [00:01<00:03, 15.19it/s]
Fetching GFS data: 32%|███▏ | 24/74 [00:01<00:03, 16.04it/s]
Fetching GFS data: 53%|█████▎ | 39/74 [00:01<00:01, 32.43it/s]
Fetching GFS data: 73%|███████▎ | 54/74 [00:02<00:00, 48.87it/s]
Fetching GFS data: 97%|█████████▋| 72/74 [00:02<00:00, 69.07it/s]
Fetching GFS data: 100%|██████████| 74/74 [00:02<00:00, 32.38it/s]
2026-08-15 05:16:01.342 | INFO | __main__:corrdiff_on_hens_ensemble:101 - Starting 2 member ensemble inference with 2 batches.
Ensemble Batches: 0%| | 0/2 [00:00<?, ?it/s]
Fetching GFS data: 0%| | 0/296 [00:00<?, ?it/s]
Fetching GFS data: 0%| | 1/296 [00:00<01:35, 3.08it/s]
Fetching GFS data: 7%|▋ | 22/296 [00:00<00:05, 54.66it/s]
Fetching GFS data: 10%|█ | 30/296 [00:00<00:05, 49.67it/s]
Fetching GFS data: 12%|█▏ | 36/296 [00:00<00:05, 50.56it/s]
Fetching GFS data: 14%|█▍ | 42/296 [00:00<00:05, 47.02it/s]
Fetching GFS data: 16%|█▌ | 48/296 [00:01<00:06, 39.36it/s]
Fetching GFS data: 18%|█▊ | 53/296 [00:01<00:08, 28.83it/s]
Fetching GFS data: 21%|██ | 61/296 [00:01<00:06, 34.56it/s]
Fetching GFS data: 23%|██▎ | 68/296 [00:01<00:07, 32.47it/s]
Fetching GFS data: 27%|██▋ | 80/296 [00:02<00:06, 31.34it/s]
Fetching GFS data: 30%|███ | 90/296 [00:02<00:05, 37.23it/s]
Fetching GFS data: 34%|███▍ | 100/296 [00:02<00:05, 38.37it/s]
Fetching GFS data: 41%|████ | 122/296 [00:03<00:03, 47.93it/s]
Fetching GFS data: 45%|████▍ | 132/296 [00:03<00:03, 49.94it/s]
Fetching GFS data: 51%|█████ | 151/296 [00:03<00:02, 49.52it/s]
Fetching GFS data: 53%|█████▎ | 157/296 [00:03<00:02, 49.18it/s]
Fetching GFS data: 56%|█████▋ | 167/296 [00:03<00:02, 56.67it/s]
Fetching GFS data: 59%|█████▉ | 176/296 [00:04<00:02, 54.24it/s]
Fetching GFS data: 64%|██████▍ | 189/296 [00:04<00:02, 51.19it/s]
Fetching GFS data: 67%|██████▋ | 197/296 [00:04<00:02, 45.19it/s]
Fetching GFS data: 74%|███████▍ | 219/296 [00:04<00:01, 50.42it/s]
Fetching GFS data: 76%|███████▌ | 225/296 [00:05<00:01, 45.00it/s]
Fetching GFS data: 78%|███████▊ | 230/296 [00:05<00:01, 44.72it/s]
Fetching GFS data: 87%|████████▋ | 258/296 [00:05<00:00, 52.58it/s]
Fetching GFS data: 90%|████████▉ | 265/296 [00:05<00:00, 54.47it/s]
Fetching GFS data: 98%|█████████▊| 290/296 [00:05<00:00, 79.42it/s]
Fetching GFS data: 100%|██████████| 296/296 [00:05<00:00, 49.72it/s]
Batch 0 inference: 0%| | 0/5 [00:00<?, ?it/s]
Batch 0 inference: 20%|██ | 1/5 [00:03<00:12, 3.24s/it]
Batch 0 inference: 40%|████ | 2/5 [00:05<00:07, 2.47s/it]
Batch 0 inference: 60%|██████ | 3/5 [00:07<00:04, 2.25s/it]
Batch 0 inference: 80%|████████ | 4/5 [00:09<00:02, 2.13s/it]
Batch 0 inference: 100%|██████████| 5/5 [00:11<00:00, 2.07s/it]
Ensemble Batches: 50%|█████ | 1/2 [00:24<00:24, 24.24s/it]
Batch 1 inference: 0%| | 0/5 [00:00<?, ?it/s]
Batch 1 inference: 20%|██ | 1/5 [00:01<00:05, 1.48s/it]
Batch 1 inference: 40%|████ | 2/5 [00:03<00:05, 1.77s/it]
Batch 1 inference: 60%|██████ | 3/5 [00:05<00:03, 1.86s/it]
Batch 1 inference: 80%|████████ | 4/5 [00:07<00:01, 1.90s/it]
Batch 1 inference: 100%|██████████| 5/5 [00:09<00:00, 1.93s/it]
Ensemble Batches: 100%|██████████| 2/2 [00:33<00:00, 15.49s/it]
Ensemble Batches: 100%|██████████| 2/2 [00:33<00:00, 16.81s/it]
2026-08-15 05:16:34.956 | SUCCESS | __main__:corrdiff_on_hens_ensemble:140 - Inference completePost-processing¶
Plot the mean and standard deviation of 10m wind speed magnitude for a sequence of lead times.
import cartopy.crs as ccrs
import matplotlib.pyplot as plt
import xarray as xr
ds = xr.open_zarr("outputs/18_ensemble_corrdiff.zarr")
lead_time = 4
arr = np.sqrt(ds["u10m"] ** 2 + ds["v10m"] ** 2)
mean_field = arr.mean(dim=["ensemble", "sample"])
std_field = arr.std(dim=["ensemble", "sample"])
fig, ax = plt.subplots(
2, lead_time, figsize=(12, 5), subplot_kw={"projection": ccrs.PlateCarree()}
)
for i in range(lead_time):
p1 = ax[0, i].contourf(
ds["lon"],
ds["lat"],
mean_field.isel(time=0, lead_time=i),
levels=20,
vmin=0,
vmax=40,
transform=ccrs.PlateCarree(),
cmap="nipy_spectral",
)
ax[0, i].coastlines()
ax[0, i].set_title(f"lead_time={6*i}hr")
p2 = ax[1, i].contourf(
ds["lon"],
ds["lat"],
std_field.isel(time=0, lead_time=i),
levels=20,
vmin=0,
vmax=4,
transform=ccrs.PlateCarree(),
cmap="magma",
)
ax[1, i].coastlines()
fig.colorbar(p1, ax=ax[0, -1], label="wind speed mean")
fig.colorbar(p2, ax=ax[1, -1], label="wind speed std")
fig.suptitle(
f"Start date: {np.datetime_as_string(ds['time'].values[0], unit='h')}", fontsize=12
)
# Leave room for suptitle
plt.tight_layout(rect=[0, 0, 1, 0.95])
plt.savefig("outputs/18_ensemble_corrdiff_w10m.jpg")
plt.show()
Execution profile
Runtime telemetry
Total runtime4m 39s
Execution environment
CPUAMD EPYC 7313P 16-Core Processor
GPUNVIDIA H100 PCIe · 79.6 GiB
System RAM58.5 GiB
PlatformLinux 6.8.0-136-generic
Python3.13.13
GPU driver / CUDADriver 595.84 · CUDA support 13.2