Running Ensemble Inference¶
Simple ensemble inference workflow.
This example will demonstrate how to run a simple inference workflow to generate a ensemble forecast using one of the built in models of Earth-2 Inference Studio.
In this example you will learn:
- How to instantiate a built in prognostic model
- Creating a data source and IO object
- Select a perturbation method
- Running a simple built in workflow for ensembling
- Post-processing results
Set Up¶
All workflows inside Earth2Studio require constructed components to be
handed to them. In this example, we will use the built in ensemble workflow
earth2studio.run.ensemble.
# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES.
# SPDX-FileCopyrightText: All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import OrderedDict
from datetime import datetime
from math import ceil
import numpy as np
import torch
from loguru import logger
from tqdm import tqdm
from earth2studio.data import DataSource, ForecastSource, fetch_data
from earth2studio.io import IOBackend
from earth2studio.models.dx import DiagnosticModel
from earth2studio.models.px import PrognosticModel
from earth2studio.perturbation import Perturbation
from earth2studio.utils.checkpoint import (
Checkpoint,
CheckpointSession,
NullCheckpoint,
)
from earth2studio.utils.coords import CoordSystem, map_coords, split_coords
from earth2studio.utils.time import to_time_array
logger.remove()
logger.add(lambda msg: tqdm.write(msg, end=""), colorize=True)
def deterministic(
time: list[str] | list[datetime] | list[np.datetime64],
nsteps: int,
prognostic: PrognosticModel,
data: DataSource,
io: IOBackend,
output_coords: CoordSystem = OrderedDict({}),
device: torch.device | None = None,
verbose: bool = True,
checkpoint: Checkpoint | CheckpointSession | NullCheckpoint = NullCheckpoint(),
) -> IOBackend:
"""Built in deterministic workflow.
This workflow creates a determinstic inference pipeline to produce a forecast
prediction using a prognostic model.
Parameters
----------
time : list[str] | list[datetime] | list[np.datetime64]
List of string, datetimes or np.datetime64
nsteps : int
Number of forecast steps
prognostic : PrognosticModel
Prognostic model
data : DataSource
Data source
io : IOBackend
IO object
output_coords: CoordSystem, optional
IO output coordinate system override, by default OrderedDict({})
device : torch.device, optional
Device to run inference on, by default None
verbose : bool, optional
Print inference progress, by default True
checkpoint : Checkpoint, optional
Checkpoint manager or checkpoint session used to record and resume workflow
progress, by default no checkpoint
Returns
-------
IOBackend
Output IO object
"""
logger.info("Running simple workflow!")
# Load model onto the device
device = (
device
if device is not None
else torch.device("cuda" if torch.cuda.is_available() else "cpu")
)
logger.info(f"Inference device: {device}")
prognostic = prognostic.to(device)
prognostic_ic = prognostic.input_coords()
time = to_time_array(time)
# Set up IO backend
total_coords = prognostic.output_coords(prognostic.input_coords()).copy()
for key, value in prognostic.output_coords(
prognostic.input_coords()
).items(): # Scrub batch dims
if value.shape == (0,):
del total_coords[key]
total_coords["time"] = time
total_coords["lead_time"] = np.asarray(
[
prognostic.output_coords(prognostic.input_coords())["lead_time"] * i
for i in range(nsteps + 1)
]
).flatten()
total_coords.move_to_end("lead_time", last=False)
total_coords.move_to_end("time", last=False)
for key, value in total_coords.items():
total_coords[key] = output_coords.get(key, value)
var_names = total_coords.pop("variable")
io.add_array(total_coords, var_names)
with checkpoint as ckpt:
restart_step = None
if ckpt.exists and ckpt.write_count > 0:
if ckpt.catalog.level < 2:
logger.warning(
"deterministic received checkpoint level "
f"{ckpt.catalog.level}; component state may not be "
"complete enough to resume a rollout. Re-running from "
"lead time zero."
)
else:
restart_step = ckpt.write_count - 1
if restart_step >= nsteps:
logger.success("\nInference complete")
return io
# Fetch data from data source and load onto device
if hasattr(prognostic, "interp_method"):
interp_to = prognostic_ic
interp_method = prognostic.interp_method
else:
interp_to = None
interp_method = "nearest"
x, coords = fetch_data(
source=data,
time=time,
variable=prognostic_ic["variable"],
lead_time=prognostic_ic["lead_time"],
device=device,
interp_to=interp_to,
interp_method=interp_method,
)
logger.success(f"Fetched data from {data.__class__.__name__}")
# Map lat and lon if needed
x, coords = map_coords(x, coords, prognostic.input_coords())
# Create prognostic iterator
model = prognostic.create_iterator(x, coords)
logger.info("Inference starting!")
initial_progress = 0 if restart_step is None else restart_step + 1
with tqdm(
total=nsteps + 1,
initial=initial_progress,
desc="Running inference",
position=1,
disable=(not verbose),
) as pbar:
for local_step, (x, coords) in enumerate(model):
step = (
local_step
if restart_step is None
else restart_step + local_step + 1
)
current_lead_time = coords["lead_time"][-1]
# Subselect domain/variables as indicated in output_coords
x, coords = map_coords(x, coords, output_coords)
io.write(*split_coords(x, coords))
ckpt.write(lead_time=current_lead_time)
pbar.update(1)
if step == nsteps:
break
ckpt.flush()
logger.success("\nInference complete")
return io
def diagnostic(
time: list[str] | list[datetime] | list[np.datetime64],
nsteps: int,
prognostic: PrognosticModel,
diagnostic: DiagnosticModel,
data: DataSource | ForecastSource,
io: IOBackend,
output_coords: CoordSystem = OrderedDict({}),
device: torch.device | None = None,
verbose: bool = True,
checkpoint: Checkpoint | CheckpointSession | NullCheckpoint = NullCheckpoint(),
) -> IOBackend:
"""Built in diagnostic workflow.
This workflow creates a determinstic inference pipeline that couples a prognostic
model with a diagnostic model.
Parameters
----------
time : list[str] | list[datetime] | list[np.datetime64]
List of string, datetimes or np.datetime64
nsteps : int
Number of forecast steps
prognostic : PrognosticModel
Prognostic model
diagnostic: DiagnosticModel
Diagnostic model, must be on same coordinate axis as prognostic
data : DataSource | ForecastSource
Data source
io : IOBackend
IO object
output_coords: CoordSystem, optional
IO output coordinate system override, by default OrderedDict({})
device : torch.device, optional
Device to run inference on, by default None
verbose : bool, optional
Print inference progress, by default True
checkpoint : Checkpoint, optional
Checkpoint manager or checkpoint session used to record and resume workflow
progress, by default no checkpoint
Returns
-------
IOBackend
Output IO object
"""
logger.info("Running diagnostic workflow!")
device = (
device
if device is not None
else torch.device("cuda" if torch.cuda.is_available() else "cpu")
)
logger.info(f"Inference device: {device}")
prognostic = prognostic.to(device)
diagnostic = diagnostic.to(device)
prognostic_ic = prognostic.input_coords()
diagnostic_ic = diagnostic.input_coords()
time = to_time_array(time)
total_coords = prognostic.output_coords(prognostic.input_coords())
for key, value in prognostic.output_coords(
prognostic.input_coords()
).items(): # Scrub batch dims
if key in diagnostic.output_coords(diagnostic_ic):
total_coords[key] = diagnostic.output_coords(diagnostic_ic)[key]
if value.shape == (0,):
del total_coords[key]
total_coords["time"] = time
total_coords["lead_time"] = np.asarray(
[
prognostic.output_coords(prognostic.input_coords())["lead_time"] * i
for i in range(nsteps + 1)
]
).flatten()
total_coords.move_to_end("lead_time", last=False)
total_coords.move_to_end("time", last=False)
for key, value in total_coords.items():
total_coords[key] = output_coords.get(key, value)
var_names = total_coords.pop("variable")
io.add_array(total_coords, var_names)
with checkpoint as ckpt:
restart_step = None
if ckpt.exists and ckpt.write_count > 0:
if ckpt.catalog.level < 2:
logger.warning(
"diagnostic received checkpoint level "
f"{ckpt.catalog.level}; component state may not be "
"complete enough to resume a rollout. Re-running from "
"lead time zero."
)
else:
restart_step = ckpt.write_count - 1
if restart_step >= nsteps:
logger.success("\nInference complete")
return io
if hasattr(prognostic, "interp_method"):
interp_to = prognostic_ic
interp_method = prognostic.interp_method
else:
interp_to = None
interp_method = "nearest"
x, coords = fetch_data(
source=data,
time=time,
variable=prognostic_ic["variable"],
lead_time=prognostic_ic["lead_time"],
device=device,
interp_to=interp_to,
interp_method=interp_method,
)
logger.success(f"Fetched data from {data.__class__.__name__}")
x, coords = map_coords(x, coords, prognostic_ic)
model = prognostic.create_iterator(x, coords)
logger.info("Inference starting!")
initial_progress = 0 if restart_step is None else restart_step + 1
with tqdm(
total=nsteps + 1,
initial=initial_progress,
desc="Running inference",
position=1,
disable=(not verbose),
) as pbar:
for local_step, (x, coords) in enumerate(model):
step = (
local_step
if restart_step is None
else restart_step + local_step + 1
)
current_lead_time = coords["lead_time"][-1]
x, coords = map_coords(x, coords, diagnostic_ic)
x, coords = diagnostic(x, coords)
x, coords = map_coords(x, coords, output_coords)
io.write(*split_coords(x, coords))
ckpt.write(lead_time=current_lead_time)
pbar.update(1)
if step == nsteps:
break
ckpt.flush()
logger.success("\nInference complete")
return io
def ensemble(
time: list[str] | list[datetime] | list[np.datetime64],
nsteps: int,
nensemble: int,
prognostic: PrognosticModel,
data: DataSource,
io: IOBackend,
perturbation: Perturbation,
batch_size: int | None = None,
output_coords: CoordSystem = OrderedDict({}),
device: torch.device | None = None,
verbose: bool = True,
checkpoint: Checkpoint | CheckpointSession | NullCheckpoint = NullCheckpoint(),
) -> IOBackend:
"""Built in ensemble workflow.
Parameters
----------
time : list[str] | list[datetime] | list[np.datetime64]
List of string, datetimes or np.datetime64
nsteps : int
Number of forecast steps
nensemble : int
Number of ensemble members to run inference for.
prognostic : PrognosticModel
Prognostic models
data : DataSource
Data source
io : IOBackend
IO object
perturbation : Perturbation
Method to perturb the initial condition to create an ensemble.
batch_size: int, optional
Number of ensemble members to run in a single batch,
by default None.
output_coords: CoordSystem, optional
IO output coordinate system override, by default OrderedDict({})
device : torch.device, optional
Device to run inference on, by default None
verbose : bool, optional
Print inference progress, by default True
checkpoint : Checkpoint, optional
Checkpoint manager or checkpoint session used to record and resume workflow
progress, by default no checkpoint
Returns
-------
IOBackend
Output IO object
"""
logger.info("Running ensemble inference!")
device = (
device
if device is not None
else torch.device("cuda" if torch.cuda.is_available() else "cpu")
)
logger.info(f"Inference device: {device}")
prognostic = prognostic.to(device)
prognostic_ic = prognostic.input_coords()
time = to_time_array(time)
if hasattr(prognostic, "interp_method"):
interp_to = prognostic_ic
interp_method = prognostic.interp_method
else:
interp_to = None
interp_method = "nearest"
x0, coords0 = fetch_data(
source=data,
time=time,
variable=prognostic_ic["variable"],
lead_time=prognostic_ic["lead_time"],
device=device,
interp_to=interp_to,
interp_method=interp_method,
)
logger.success(f"Fetched data from {data.__class__.__name__}")
total_coords = prognostic.output_coords(prognostic.input_coords()).copy()
if "batch" in total_coords:
del total_coords["batch"]
total_coords["time"] = time
total_coords["lead_time"] = np.asarray(
[
prognostic.output_coords(prognostic.input_coords())["lead_time"] * i
for i in range(nsteps + 1)
]
).flatten()
total_coords.move_to_end("lead_time", last=False)
total_coords.move_to_end("time", last=False)
total_coords = {"ensemble": np.arange(nensemble)} | total_coords
for key, value in total_coords.items():
total_coords[key] = output_coords.get(key, value)
variables_to_save = total_coords.pop("variable")
io.add_array(total_coords, variables_to_save)
if batch_size is None:
batch_size = nensemble
batch_size = min(nensemble, batch_size)
with checkpoint as ckpt:
completed_ensembles = []
if ckpt.exists and not isinstance(ckpt, NullCheckpoint):
completed_ensembles = [
int(value) for value in ckpt.metadata.get("completed_ensembles", [])
]
completed = set(completed_ensembles)
start_batch_id = next(
(index for index in range(nensemble) if index not in completed),
nensemble,
)
number_of_batches = ceil((nensemble - start_batch_id) / batch_size)
restart_first_batch = (
ckpt.exists
and ckpt.write_count > 0
and start_batch_id < nensemble
and ckpt.lead_time != total_coords["lead_time"][-1]
)
logger.info(f"Starting {nensemble} Member Ensemble Inference with \
{number_of_batches} number of batches.")
for batch_index, batch_id in enumerate(
tqdm(
range(start_batch_id, nensemble, batch_size),
total=number_of_batches,
desc="Total Ensemble Batches",
position=2,
disable=(not verbose),
)
):
mini_batch_size = min(batch_size, nensemble - batch_id)
ensemble_coords = np.arange(batch_id, batch_id + mini_batch_size)
ensemble_members = [int(value) for value in ensemble_coords]
restart_step = None
if batch_index == 0 and restart_first_batch:
if ckpt.catalog.level < 2:
logger.warning(
"ensemble received checkpoint level "
f"{ckpt.catalog.level}; component state may not be "
"complete enough to resume a rollout. Re-running from "
"lead time zero."
)
ckpt.write_count = 0
else:
restart_step = ckpt.write_count - 1
if restart_step >= nsteps:
continue
elif not isinstance(ckpt, NullCheckpoint):
ckpt.write_count = 0
x = x0.to(device)
coords = OrderedDict({"ensemble": ensemble_coords}) | coords0.copy()
x = x.unsqueeze(0).repeat(mini_batch_size, *([1] * x.ndim))
x, coords = map_coords(x, coords, prognostic_ic)
x, coords = perturbation(x, coords)
model = prognostic.create_iterator(x, coords)
initial_progress = 0 if restart_step is None else restart_step + 1
with tqdm(
total=nsteps + 1,
initial=initial_progress,
desc=f"Running batch {batch_id} inference",
position=1,
leave=False,
disable=(not verbose),
) as pbar:
for local_step, (x, coords) in enumerate(model):
step = (
local_step
if restart_step is None
else restart_step + local_step + 1
)
current_lead_time = coords["lead_time"][-1]
x, coords = map_coords(x, coords, output_coords)
io.write(*split_coords(x, coords))
if step == nsteps:
completed.update(ensemble_members)
completed_ensembles = sorted(completed)
ckpt.write(
lead_time=current_lead_time,
completed_ensembles=completed_ensembles,
)
pbar.update(1)
if step == nsteps:
break
ckpt.flush()
logger.success("\nInference complete")
return io
We need the following:
- Prognostic Model: Use the built in FourCastNet model
earth2studio.models.px.FCN. - Perturbation Method: Use the Spherical Gaussian Method
earth2studio.perturbation.SphericalGaussian. - Datasource: Pull data from the GFS data api
earth2studio.data.GFS. - IO Backend: Save the outputs into a Zarr store
earth2studio.io.ZarrBackend.
import os
os.makedirs("outputs", exist_ok=True)
from dotenv import load_dotenv
load_dotenv() # TODO: make common example prep function
import numpy as np
from earth2studio.data import GFS
from earth2studio.io import ZarrBackend
from earth2studio.models.px import FCN
from earth2studio.perturbation import SphericalGaussian
from earth2studio.run import ensemble
from earth2studio.utils.time import to_time_array
# Load the default model package which downloads the check point from NGC
package = FCN.load_default_package()
model = FCN.load_model(package)
# Instantiate the pertubation method
sg = SphericalGaussian(noise_amplitude=0.15)
# Create the data source
data = GFS()
# Create the IO handler, store in memory
chunks = {"ensemble": 1, "time": 1, "lead_time": 1}
io = ZarrBackend(
file_name="outputs/03_ensemble_sg.zarr",
chunks=chunks,
backend_kwargs={"overwrite": True},
)
Console output9 lines
/__w/earth2studio/earth2studio/.venv/lib/python3.13/site-packages/torch/cuda/__init__.py:64: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
import pynvml # type: ignore[import]
WARNING[XFORMERS]: xFormers can't load C++/CUDA extensions. xFormers was built for:
PyTorch 2.10.0+cu128 with CUDA 1208 (you have 2.13.0+cu130)
Python 3.10.19 (you have 3.13.13)
Please reinstall xformers (see https://github.com/facebookresearch/xformers#installing-xformers)
Memory-efficient attention, SwiGLU, sparse and more won't be available.
Set XFORMERS_MORE_DETAILS=1 for more details
CuPy distance computation test failed with error: cuVS >= 24.12 or pylibraft < 24.12 should be installed to use this featureFetch Data¶
You can easily fetch raw Xarray data from an initial condition data source with a simple call. By default, this caches the data locally on your machine, so you won't have to re-download it if you access it again or use it in an inference pipeline.
sample = data(
to_time_array(["2024-01-01"]),
model.input_coords()["variable"],
)
print(f"Cached GFS input shape: {sample.shape}")
Console output4 lines
Fetching GFS data: 0%| | 0/26 [00:00<?, ?it/s]
Fetching GFS data: 4%|▍ | 1/26 [00:00<00:19, 1.26it/s]
Fetching GFS data: 100%|██████████| 26/26 [00:00<00:00, 31.29it/s]
Cached GFS input shape: (1, 26, 721, 1440)Execute the Workflow¶
With all components initialized, running the workflow is a single line of Python code. Workflow will return the provided IO object back to the user, which can be used to then post process. Some have additional APIs that can be handy for post-processing or saving to file. Check the API docs for more information.
For the forecast we will predict for 10 steps (for FCN, this is 60 hours) with 8 ensemble members which will be ran in 2 batches with batch size 4.
nsteps = 10
nensemble = 8
batch_size = 2
io = ensemble(
["2024-01-01"],
nsteps,
nensemble,
model,
data,
io,
sg,
batch_size=batch_size,
output_coords={"variable": np.array(["t2m", "tcwv"])},
)
Console output131 lines
2026-08-15 04:36:47.079 | INFO | earth2studio.run:ensemble:394 - Running ensemble inference!
2026-08-15 04:36:47.079 | INFO | earth2studio.run:ensemble:401 - Inference device: cuda
Fetching GFS data: 0%| | 0/26 [00:00<?, ?it/s]
Fetching GFS data: 4%|▍ | 1/26 [00:00<00:17, 1.42it/s]
Fetching GFS data: 100%|██████████| 26/26 [00:00<00:00, 36.15it/s]
2026-08-15 04:36:47.956 | SUCCESS | earth2studio.run:ensemble:422 - Fetched data from GFS
2026-08-15 04:36:48.036 | INFO | earth2studio.run:ensemble:466 - Starting 8 Member Ensemble Inference with 4 number of batches.
Total Ensemble Batches: 0%| | 0/4 [00:00<?, ?it/s]
Running batch 0 inference: 0%| | 0/11 [00:00<?, ?it/s]
Running batch 0 inference: 9%|▉ | 1/11 [00:00<00:02, 3.97it/s]
Running batch 0 inference: 18%|█▊ | 2/11 [00:00<00:03, 2.68it/s]
Running batch 0 inference: 27%|██▋ | 3/11 [00:01<00:02, 2.93it/s]
Running batch 0 inference: 36%|███▋ | 4/11 [00:01<00:02, 3.04it/s]
Running batch 0 inference: 45%|████▌ | 5/11 [00:01<00:01, 3.10it/s]
Running batch 0 inference: 55%|█████▍ | 6/11 [00:01<00:01, 3.19it/s]
Running batch 0 inference: 64%|██████▎ | 7/11 [00:02<00:01, 3.16it/s]
Running batch 0 inference: 73%|███████▎ | 8/11 [00:02<00:00, 3.28it/s]
Running batch 0 inference: 82%|████████▏ | 9/11 [00:02<00:00, 3.41it/s]
Running batch 0 inference: 91%|█████████ | 10/11 [00:03<00:00, 3.39it/s]
Running batch 0 inference: 100%|██████████| 11/11 [00:03<00:00, 3.49it/s]
Total Ensemble Batches: 25%|██▌ | 1/4 [00:04<00:14, 4.87s/it]
Running batch 2 inference: 0%| | 0/11 [00:00<?, ?it/s]
Running batch 2 inference: 9%|▉ | 1/11 [00:00<00:01, 5.79it/s]
Running batch 2 inference: 18%|█▊ | 2/11 [00:00<00:02, 4.47it/s]
Running batch 2 inference: 27%|██▋ | 3/11 [00:00<00:02, 3.90it/s]
Running batch 2 inference: 36%|███▋ | 4/11 [00:01<00:01, 3.78it/s]
Running batch 2 inference: 45%|████▌ | 5/11 [00:01<00:01, 3.78it/s]
Running batch 2 inference: 55%|█████▍ | 6/11 [00:01<00:01, 3.79it/s]
Running batch 2 inference: 64%|██████▎ | 7/11 [00:01<00:01, 3.77it/s]
Running batch 2 inference: 73%|███████▎ | 8/11 [00:02<00:00, 3.81it/s]
Running batch 2 inference: 82%|████████▏ | 9/11 [00:02<00:00, 3.81it/s]
Running batch 2 inference: 91%|█████████ | 10/11 [00:02<00:00, 3.80it/s]
Running batch 2 inference: 100%|██████████| 11/11 [00:02<00:00, 3.79it/s]
Total Ensemble Batches: 50%|█████ | 2/4 [00:08<00:08, 4.39s/it]
Running batch 4 inference: 0%| | 0/11 [00:00<?, ?it/s]
Running batch 4 inference: 9%|▉ | 1/11 [00:00<00:02, 4.92it/s]
Running batch 4 inference: 18%|█▊ | 2/11 [00:00<00:02, 4.07it/s]
Running batch 4 inference: 27%|██▋ | 3/11 [00:00<00:02, 3.75it/s]
Running batch 4 inference: 36%|███▋ | 4/11 [00:01<00:01, 3.78it/s]
Running batch 4 inference: 45%|████▌ | 5/11 [00:01<00:01, 3.82it/s]
Running batch 4 inference: 55%|█████▍ | 6/11 [00:01<00:01, 3.74it/s]
Running batch 4 inference: 64%|██████▎ | 7/11 [00:01<00:01, 3.72it/s]
Running batch 4 inference: 73%|███████▎ | 8/11 [00:02<00:00, 3.68it/s]
Running batch 4 inference: 82%|████████▏ | 9/11 [00:02<00:00, 3.59it/s]
Running batch 4 inference: 91%|█████████ | 10/11 [00:02<00:00, 3.50it/s]
Running batch 4 inference: 100%|██████████| 11/11 [00:03<00:00, 3.49it/s]
Total Ensemble Batches: 75%|███████▌ | 3/4 [00:13<00:04, 4.31s/it]
Running batch 6 inference: 0%| | 0/11 [00:00<?, ?it/s]
Running batch 6 inference: 9%|▉ | 1/11 [00:00<00:01, 5.28it/s]
Running batch 6 inference: 18%|█▊ | 2/11 [00:00<00:02, 4.22it/s]
Running batch 6 inference: 27%|██▋ | 3/11 [00:00<00:02, 3.98it/s]
Running batch 6 inference: 36%|███▋ | 4/11 [00:00<00:01, 3.87it/s]
Running batch 6 inference: 45%|████▌ | 5/11 [00:01<00:01, 3.81it/s]
Running batch 6 inference: 55%|█████▍ | 6/11 [00:01<00:01, 3.77it/s]
Running batch 6 inference: 64%|██████▎ | 7/11 [00:01<00:01, 3.72it/s]
Running batch 6 inference: 73%|███████▎ | 8/11 [00:02<00:00, 3.58it/s]
Running batch 6 inference: 82%|████████▏ | 9/11 [00:02<00:00, 3.51it/s]
Running batch 6 inference: 91%|█████████ | 10/11 [00:02<00:00, 3.49it/s]
Running batch 6 inference: 100%|██████████| 11/11 [00:02<00:00, 3.46it/s]
Total Ensemble Batches: 100%|██████████| 4/4 [00:17<00:00, 4.27s/it]
Total Ensemble Batches: 100%|██████████| 4/4 [00:17<00:00, 4.34s/it]
2026-08-15 04:37:05.386 | SUCCESS | earth2studio.run:ensemble:536 -
Inference completePost Processing¶
The last step is to post process our results. Cartopy is a great library for plotting fields on projections of a sphere.
Notice that the Zarr IO function has additional APIs to interact with the stored data.
import cartopy.crs as ccrs
import matplotlib.pyplot as plt
forecast = "2024-01-01"
def plot_(axi, data, title, cmap):
"""Convenience function for plotting pcolormesh."""
# Plot the field using pcolormesh
im = axi.pcolormesh(
io["lon"][:],
io["lat"][:],
data,
transform=ccrs.PlateCarree(),
cmap=cmap,
)
plt.colorbar(im, ax=axi, shrink=0.6, pad=0.04)
# Set title
axi.set_title(title)
# Add coastlines and gridlines
axi.coastlines()
axi.gridlines()
for variable, cmap in zip(["tcwv"], ["Blues"]):
step = 4 # lead time = 24 hrs
plt.close("all")
# Create a Robinson projection
projection = ccrs.Robinson()
# Create a figure and axes with the specified projection
fig, (ax1, ax2, ax3) = plt.subplots(
nrows=1, ncols=3, subplot_kw={"projection": projection}, figsize=(16, 3)
)
plot_(
ax1,
io[variable][0, 0, step],
f"{forecast} - Lead time: {6*step}hrs - Member: {0}",
cmap,
)
plot_(
ax2,
io[variable][1, 0, step],
f"{forecast} - Lead time: {6*step}hrs - Member: {1}",
cmap,
)
plot_(
ax3,
np.std(io[variable][:, 0, step], axis=0),
f"{forecast} - Lead time: {6*step}hrs - Std",
cmap,
)
plt.savefig(f"outputs/03_{forecast}_{variable}_{step}_ensemble.jpg")
Execution profile
Runtime telemetry
Profiled phases
Total runtime1m 29s
Setup
Duration28.9 sTagged cells
CPU load10%Average · Peak 44%
Process memory2.3 GiBPeak resident set
Network received0 BHost-wide estimate
Network sent0 BHost-wide estimate
GPU utilization0%Peak 0%
GPU memory0.5 GiBof 79.6 GiB
GPU power84 WPeak draw
Inference
Duration18.3 sTagged cells
CPU load26%Average · Peak 83%
Process memory6.2 GiBPeak resident set
Network received0 BHost-wide estimate
Network sent0 BHost-wide estimate
GPU utilization14%Peak 73%
GPU memory3.8 GiBof 79.6 GiB
GPU power169 WPeak draw
Plotting
Duration37.6 sTagged cells
CPU load11%Average · Peak 14%
Process memory6.0 GiBPeak resident set
Network received70 BHost-wide estimate
Network sent0 BHost-wide estimate
GPU utilization0%Peak 0%
GPU memory3.8 GiBof 79.6 GiB
GPU power150 WPeak draw
Execution environment
CPUAMD EPYC 7313P 16-Core Processor
GPUNVIDIA H100 PCIe · 79.6 GiB
System RAM58.5 GiB
PlatformLinux 6.8.0-136-generic
Python3.13.13
GPU driver / CUDADriver 595.84 · CUDA support 13.2