Source code for multistorageclient.shortcuts

  1# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  2# SPDX-License-Identifier: Apache-2.0
  3#
  4# Licensed under the Apache License, Version 2.0 (the "License");
  5# you may not use this file except in compliance with the License.
  6# You may obtain a copy of the License at
  7#
  8# http://www.apache.org/licenses/LICENSE-2.0
  9#
 10# Unless required by applicable law or agreed to in writing, software
 11# distributed under the License is distributed on an "AS IS" BASIS,
 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13# See the License for the specific language governing permissions and
 14# limitations under the License.
 15
 16import builtins
 17import logging
 18import os
 19import re
 20import threading
 21from collections.abc import Callable, Iterator
 22from typing import Any
 23from urllib.parse import ParseResult, urlparse
 24
 25from .client import StorageClient
 26from .config import RESERVED_POSIX_PROFILE_NAME, SUPPORTED_IMPLICIT_PROFILE_PROTOCOLS, PathMapping, StorageClientConfig
 27from .file import ObjectFile, PosixFile
 28from .telemetry import Telemetry
 29from .types import MSC_PROTOCOL, ExecutionMode, ObjectMetadata, PatternList, SignerType, SymlinkHandling, SyncResult
 30
 31_TELEMETRY_PROVIDER: Callable[[], Telemetry] | None = None
 32_TELEMETRY_PROVIDER_LOCK = threading.Lock()
 33_STORAGE_CLIENT_CACHE: dict[str, StorageClient] = {}
 34_STORAGE_CLIENT_CACHE_LOCK = threading.Lock()
 35_PATH_MAPPING_CACHE: dict[str | None, PathMapping] = {}
 36_PATH_MAPPING_CACHE_LOCK = threading.Lock()
 37_PROCESS_ID = os.getpid()
 38
 39logger = logging.getLogger(__name__)
 40
 41
 42def _reinitialize_after_fork() -> None:
 43    """
 44    Reinitialize module state after fork to ensure fork-safety.
 45
 46    This function is called automatically after a fork to:
 47    1. Clear the storage client cache (cached clients may have invalid state)
 48    2. Reinitialize locks (parent's lock state must not be inherited)
 49    3. Update process ID tracking
 50
 51    Note: The telemetry provider is intentionally inherited by child processes,
 52    only its lock is reinitialized.
 53    """
 54    global _STORAGE_CLIENT_CACHE, _STORAGE_CLIENT_CACHE_LOCK  # noqa: PLW0602
 55    global _PATH_MAPPING_CACHE, _PATH_MAPPING_CACHE_LOCK  # noqa: PLW0602
 56    global _TELEMETRY_PROVIDER_LOCK
 57    global _PROCESS_ID
 58
 59    _STORAGE_CLIENT_CACHE.clear()
 60    _STORAGE_CLIENT_CACHE_LOCK = threading.Lock()
 61    _PATH_MAPPING_CACHE.clear()
 62    _PATH_MAPPING_CACHE_LOCK = threading.Lock()
 63    # we don't need to reset telemetry provider as it is supposed to be a top-level Python function
 64    _TELEMETRY_PROVIDER_LOCK = threading.Lock()
 65    _PROCESS_ID = os.getpid()
 66
 67
 68def _check_and_reinitialize_if_forked() -> None:
 69    """
 70    Check if the current process is a fork and reinitialize if needed.
 71
 72    This provides fork-safety for systems where os.register_at_fork is not available
 73    or as a fallback mechanism.
 74    """
 75    global _PROCESS_ID  # noqa: PLW0602
 76
 77    current_pid = os.getpid()
 78    if current_pid != _PROCESS_ID:
 79        _reinitialize_after_fork()
 80
 81
 82if hasattr(os, "register_at_fork"):
 83    os.register_at_fork(after_in_child=_reinitialize_after_fork)
 84
 85
[docs] 86def get_telemetry_provider() -> Callable[[], Telemetry] | None: 87 """ 88 Get the function used to create :py:class:`Telemetry` instances for storage clients created by shortcuts. 89 90 :return: A function that provides a telemetry instance. 91 """ 92 global _TELEMETRY_PROVIDER # noqa: PLW0602 93 94 return _TELEMETRY_PROVIDER
95 96
[docs] 97def set_telemetry_provider(telemetry_provider: Callable[[], Telemetry] | None) -> None: 98 """ 99 Set the function used to create :py:class:`Telemetry` instances for storage clients created by shortcuts. 100 101 :param telemetry_provider: A function that provides a telemetry instance. The function must be defined at the top level of a module to work with pickling. 102 """ 103 global _TELEMETRY_PROVIDER 104 global _TELEMETRY_PROVIDER_LOCK # noqa: PLW0602 105 106 with _TELEMETRY_PROVIDER_LOCK: 107 _TELEMETRY_PROVIDER = telemetry_provider
108 109 110def _build_full_path(original_url: str, pr: ParseResult) -> str: 111 """ 112 Helper function to construct the full path from a parsed URL, including query and fragment. 113 114 :param original_url: The original URL before parsing 115 :param pr: The parsed URL result from urlparse 116 :return: The complete path including query and fragment if present 117 """ 118 path = pr.path 119 if pr.query: 120 path += "?" + pr.query 121 elif original_url.endswith("?"): 122 path += "?" # handle the glob pattern that has a trailing question mark 123 if pr.fragment: 124 path += "#" + pr.fragment 125 return path 126 127 128def _resolve_msc_url(url: str) -> tuple[str, str]: 129 """ 130 Resolve an MSC URL to a profile name and path. 131 132 :param url: The MSC URL to resolve (msc://profile/path) 133 :return: A tuple of (profile_name, path) 134 """ 135 pr = urlparse(url) 136 profile = pr.netloc 137 # Normalize only the object path so the msc:// scheme separator and profile stay intact. 138 pr = pr._replace(path=re.sub(r"/+", "/", pr.path)) 139 path = _build_full_path(url, pr) 140 path = path.removeprefix("/") 141 return profile, path 142 143 144def _read_cached_path_mapping() -> PathMapping: 145 """ 146 Read path mapping once per ``MSC_CONFIG`` value for shortcut URL resolution. 147 148 Path mapping checks happen on every non-MSC shortcut call, including POSIX paths. Caching here keeps that hot path 149 from repeatedly loading and validating the full MSC config while preserving ``StorageClientConfig.read_path_mapping`` 150 behavior for direct callers. 151 """ 152 cache_key = os.getenv("MSC_CONFIG", None) 153 if cache_key in _PATH_MAPPING_CACHE: 154 return _PATH_MAPPING_CACHE[cache_key] 155 156 with _PATH_MAPPING_CACHE_LOCK: 157 if cache_key in _PATH_MAPPING_CACHE: 158 return _PATH_MAPPING_CACHE[cache_key] 159 160 path_mapping = StorageClientConfig.read_path_mapping() 161 _PATH_MAPPING_CACHE[cache_key] = path_mapping 162 return path_mapping 163 164 165def _resolve_non_msc_url(url: str) -> tuple[str, str]: 166 """ 167 Resolve a non-MSC URL to a profile name and path. 168 169 Resolution process: 170 1. First check if MSC config exists 171 2. If config exists, check for possible path mapping 172 3. If no mapping is found, fall back to the reserved POSIX profile (``__filesystem__``) for file paths or create an implicit profile based on URL 173 174 :param url: The non-MSC URL to resolve 175 :return: A tuple of (profile_name, path) 176 """ 177 # Check if we have a valid path mapping, if so check if there is a matching mapping 178 path_mapping = _read_cached_path_mapping() 179 if path_mapping: 180 # Look for a matching mapping 181 possible_mapping = path_mapping.find_mapping(url) 182 if possible_mapping: 183 return possible_mapping # return the profile name and path 184 185 # For file paths, use the default POSIX profile 186 if url.startswith("file://"): 187 pr = urlparse(url) 188 return RESERVED_POSIX_PROFILE_NAME, _build_full_path(url, pr) 189 elif url.startswith("/"): 190 url = os.path.normpath(url) 191 return RESERVED_POSIX_PROFILE_NAME, url 192 193 # For other URL protocol, create an implicit profile name 194 pr = urlparse(url) 195 protocol = pr.scheme.lower() 196 197 # Translate relative paths to absolute paths 198 if not protocol: 199 return RESERVED_POSIX_PROFILE_NAME, os.path.realpath(url) 200 201 # Validate the protocol is supported 202 if protocol not in SUPPORTED_IMPLICIT_PROFILE_PROTOCOLS: 203 supported_protocols = ", ".join([f"{p}://" for p in SUPPORTED_IMPLICIT_PROFILE_PROTOCOLS]) 204 raise ValueError( 205 f'Unknown URL "{url}", expecting "{MSC_PROTOCOL}" or a supported protocol ({supported_protocols}) or a POSIX path' 206 ) 207 208 # Build the implicit profile name using the format _protocol-bucket 209 bucket = pr.netloc 210 if not bucket: 211 raise ValueError(f'Invalid URL "{url}", bucket name is required for {protocol}:// URLs') 212 213 profile_name = f"_{protocol}-{bucket}" 214 215 # Return normalized path with leading slash removed 216 path = pr.path 217 path = path.removeprefix("/") 218 219 return profile_name, path 220 221
[docs] 222def resolve_storage_client(url: str) -> tuple[StorageClient, str]: 223 """ 224 Build and return a :py:class:`multistorageclient.StorageClient` instance based on the provided URL or path. 225 226 This function parses the given URL or path and determines the appropriate storage profile and path. 227 It supports URLs with the protocol ``msc://``, as well as POSIX paths or ``file://`` URLs for local file 228 system access. If the profile has already been instantiated, it returns the cached client. Otherwise, 229 it creates a new :py:class:`StorageClient` and caches it. 230 231 The function also supports implicit profiles for non-MSC URLs. When a non-MSC URL is provided (like s3://, 232 gs://, ais://, file://), MSC will infer the storage provider based on the URL protocol and create an implicit 233 profile with the naming convention "_protocol-bucket" (e.g., "_s3-bucket1", "_gs-bucket1"). 234 235 Path mapping defined in the MSC configuration are also applied before creating implicit profiles. 236 This allows for explicit mappings between source paths and destination MSC profiles. 237 238 This function is fork-safe: after a fork, the cache is automatically cleared and new client instances 239 are created in the child process to avoid sharing stale connections or file descriptors. 240 241 :param url: The storage location, which can be: 242 - A URL in the format ``msc://profile/path`` for object storage. 243 - A local file system path (absolute POSIX path) or a ``file://`` URL. 244 - A non-MSC URL with a supported protocol (s3://, gs://, ais://). 245 246 :return: A tuple containing the :py:class:`multistorageclient.StorageClient` instance and the parsed path. 247 248 :raises ValueError: If the URL's protocol is neither ``msc`` nor a valid local file system path 249 or a supported non-MSC protocol. 250 """ 251 global _STORAGE_CLIENT_CACHE # noqa: PLW0602 252 global _STORAGE_CLIENT_CACHE_LOCK # noqa: PLW0602 253 254 _check_and_reinitialize_if_forked() 255 256 # Normalize the path for msc:/ prefix due to pathlib.Path('msc://') 257 if url.startswith("msc:/") and not url.startswith("msc://"): 258 url = url.replace("msc:/", "msc://") 259 260 # Resolve the URL to a profile name and path 261 profile, path = _resolve_msc_url(url) if url.startswith(MSC_PROTOCOL) else _resolve_non_msc_url(url) 262 263 # Check if the profile has already been instantiated 264 if profile in _STORAGE_CLIENT_CACHE: 265 return _STORAGE_CLIENT_CACHE[profile], path 266 267 # Create a new StorageClient instance and cache it 268 with _STORAGE_CLIENT_CACHE_LOCK: 269 if profile in _STORAGE_CLIENT_CACHE: 270 return _STORAGE_CLIENT_CACHE[profile], path 271 else: 272 client = StorageClient( 273 config=StorageClientConfig.from_file(profile=profile, telemetry_provider=get_telemetry_provider()) 274 ) 275 _STORAGE_CLIENT_CACHE[profile] = client 276 277 return client, path
278 279
[docs] 280def open(url: str, mode: str = "rb", **kwargs: Any) -> PosixFile | ObjectFile: 281 """ 282 Open a file at the given URL using the specified mode. 283 284 The function utilizes the :py:class:`multistorageclient.StorageClient` to open a file at the provided path. 285 The URL is parsed, and the corresponding :py:class:`multistorageclient.StorageClient` is retrieved or built. 286 287 :param url: The URL of the file to open. (example: ``msc://profile/prefix/dataset.tar``) 288 :param mode: The file mode to open the file in. 289 290 :return: A file-like object that allows interaction with the file. 291 292 :raises ValueError: If the URL's protocol does not match the expected protocol ``msc``. 293 """ 294 client, path = resolve_storage_client(url) 295 return client.open(path, mode, **kwargs)
296 297
[docs] 298def glob(pattern: str, attribute_filter_expression: str | None = None) -> builtins.list[str]: 299 """ 300 Return a list of files matching a pattern. 301 302 This function supports glob-style patterns for matching multiple files within a storage system. The pattern is 303 parsed, and the associated :py:class:`multistorageclient.StorageClient` is used to retrieve the 304 list of matching files. 305 306 :param pattern: The glob-style pattern to match files. (example: ``msc://profile/prefix/**/*.tar``) 307 :param attribute_filter_expression: The attribute filter expression to apply to the result. 308 309 :return: A list of file paths matching the pattern. 310 311 :raises ValueError: If the URL's protocol does not match the expected protocol ``msc``. 312 """ 313 client, path = resolve_storage_client(pattern) 314 if not pattern.startswith(MSC_PROTOCOL) and client.profile == RESERVED_POSIX_PROFILE_NAME: 315 return client.glob(path, include_url_prefix=False, attribute_filter_expression=attribute_filter_expression) 316 else: 317 return client.glob(path, include_url_prefix=True, attribute_filter_expression=attribute_filter_expression)
318 319
[docs] 320def upload_file(url: str, local_path: str, attributes: dict[str, Any] | None = None) -> None: 321 """ 322 Upload a file to the given URL from a local path. 323 324 The function utilizes the :py:class:`multistorageclient.StorageClient` to upload a file (object) to the 325 provided path. The URL is parsed, and the corresponding :py:class:`multistorageclient.StorageClient` 326 is retrieved or built. 327 328 :param url: The URL of the file. (example: ``msc://profile/prefix/dataset.tar``) 329 :param local_path: The local path of the file. 330 331 :raises ValueError: If the URL's protocol does not match the expected protocol ``msc``. 332 """ 333 client, path = resolve_storage_client(url) 334 return client.upload_file(remote_path=path, local_path=local_path, attributes=attributes)
335 336
[docs] 337def download_file(url: str, local_path: str) -> None: 338 """ 339 Download a file in a given remote_path to a local path 340 341 The function utilizes the :py:class:`multistorageclient.StorageClient` to download a file (object) at the 342 provided path. The URL is parsed, and the corresponding :py:class:`multistorageclient.StorageClient` 343 is retrieved or built. 344 345 :param url: The URL of the file to download. (example: ``msc://profile/prefix/dataset.tar``) 346 :param local_path: The local path where the file should be downloaded. 347 348 :raises ValueError: If the URL's protocol does not match the expected protocol ``msc``. 349 """ 350 client, path = resolve_storage_client(url) 351 return client.download_file(remote_path=path, local_path=local_path)
352 353
[docs] 354def is_empty(url: str) -> bool: 355 """ 356 Checks whether the specified URL contains any objects. 357 358 :param url: The URL to check, typically pointing to a storage location. 359 :return: ``True`` if there are no objects/files under this URL, ``False`` otherwise. 360 361 :raises ValueError: If the URL's protocol does not match the expected protocol ``msc``. 362 """ 363 client, path = resolve_storage_client(url) 364 return client.is_empty(path)
365 366
[docs] 367def is_file(url: str) -> bool: 368 """ 369 Checks whether the specified url points to a file (rather than a directory or folder). 370 371 The function utilizes the :py:class:`multistorageclient.StorageClient` to check if a file (object) exists 372 at the provided path. The URL is parsed, and the corresponding :py:class:`multistorageclient.StorageClient` 373 is retrieved or built. 374 375 :param url: The URL to check the existence of a file. (example: ``msc://profile/prefix/dataset.tar``) 376 """ 377 client, path = resolve_storage_client(url) 378 return client.is_file(path=path)
379 380
[docs] 381def sync( 382 source_url: str, 383 target_url: str, 384 delete_unmatched_files: bool = False, 385 execution_mode: ExecutionMode = ExecutionMode.LOCAL, 386 patterns: PatternList | None = None, 387 preserve_source_attributes: bool = False, 388 ignore_hidden: bool = True, 389 dryrun: bool = False, 390 dryrun_output_path: str | None = None, 391 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW, 392) -> SyncResult: 393 """ 394 Syncs files from the source storage to the target storage. 395 396 :param source_url: The URL for the source storage. 397 :param target_url: The URL for the target storage. 398 :param delete_unmatched_files: Whether to delete files at the target that are not present at the source. 399 :param execution_mode: The execution mode to use. Currently supports "local" and "ray". 400 :param patterns: PatternList for include/exclude filtering. If None, all files are included. 401 :param preserve_source_attributes: Whether to preserve source file metadata attributes during synchronization. 402 When False (default), only file content is copied. When True, custom metadata attributes are also preserved. 403 404 .. warning:: 405 **Performance Impact**: When enabled without a ``metadata_provider`` configured, this will make a HEAD 406 request for each object to retrieve attributes, which can significantly impact performance on large-scale 407 sync operations. For production use at scale, configure a ``metadata_provider`` in your storage profile. 408 :param ignore_hidden: Whether to ignore hidden files and directories (starting with dot). Default is True. 409 :param dryrun: If True, only enumerate and compare objects without performing any copy/delete operations. 410 The returned :py:class:`SyncResult` will include a :py:class:`DryrunResult` with paths to JSONL files. 411 :param dryrun_output_path: Directory to write dryrun JSONL files into. If None (default), a temporary 412 directory is created automatically. Ignored when dryrun is False. 413 :param symlink_handling: How to handle symbolic links during sync. 414 :py:attr:`SymlinkHandling.FOLLOW` (default) dereferences symlinks and copies the target's bytes. 415 :py:attr:`SymlinkHandling.SKIP` excludes symlinks from the sync. 416 :py:attr:`SymlinkHandling.PRESERVE` recreates symlinks on the target via 417 :py:meth:`AbstractStorageClient.make_symlink` instead of copying bytes (required for 418 round-trip preservation of symlinks). 419 """ 420 source_client, source_path = resolve_storage_client(source_url) 421 target_client, target_path = resolve_storage_client(target_url) 422 return target_client.sync_from( 423 source_client, 424 source_path, 425 target_path, 426 delete_unmatched_files, 427 execution_mode=execution_mode, 428 patterns=patterns, 429 preserve_source_attributes=preserve_source_attributes, 430 ignore_hidden=ignore_hidden, 431 dryrun=dryrun, 432 dryrun_output_path=dryrun_output_path, 433 symlink_handling=symlink_handling, 434 )
435 436
[docs] 437def sync_replicas( 438 source_url: str, 439 replica_indices: builtins.list[int] | None = None, 440 delete_unmatched_files: bool = False, 441 execution_mode: ExecutionMode = ExecutionMode.LOCAL, 442 patterns: PatternList | None = None, 443 ignore_hidden: bool = True, 444 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW, 445) -> None: 446 """ 447 Syncs files from the source storage to all the replicas. 448 449 :param source_url: The URL for the source storage. 450 :param replica_indices: Specify the indices of the replicas to sync to. If not provided, all replicas will be synced. Index starts from 0. 451 :param delete_unmatched_files: Whether to delete files at the replicas that are not present at the source. 452 :param execution_mode: The execution mode to use. Currently supports "local" and "ray". 453 :param patterns: PatternList for include/exclude filtering. If None, all files are included. 454 :param ignore_hidden: Whether to ignore hidden files and directories (starting with dot). Default is True. 455 :param symlink_handling: How to handle symbolic links during sync. 456 :py:attr:`SymlinkHandling.FOLLOW` (default) dereferences symlinks and copies the target's bytes. 457 :py:attr:`SymlinkHandling.SKIP` excludes symlinks from the sync. 458 :py:attr:`SymlinkHandling.PRESERVE` recreates symlinks on each replica via 459 :py:meth:`AbstractStorageClient.make_symlink` instead of copying bytes. 460 """ 461 source_client, source_path = resolve_storage_client(source_url) 462 source_client.sync_replicas( 463 source_path, 464 replica_indices=replica_indices, 465 delete_unmatched_files=delete_unmatched_files, 466 execution_mode=execution_mode, 467 patterns=patterns, 468 ignore_hidden=ignore_hidden, 469 symlink_handling=symlink_handling, 470 )
471 472
[docs] 473def list( 474 url: str, 475 start_after: str | None = None, 476 end_at: str | None = None, 477 include_directories: bool = False, 478 attribute_filter_expression: str | None = None, 479 show_attributes: bool = False, 480 patterns: PatternList | None = None, 481 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW, 482) -> Iterator[ObjectMetadata]: 483 """ 484 Lists the contents of the specified URL prefix. 485 486 This function retrieves the corresponding :py:class:`multistorageclient.StorageClient` 487 for the given URL and returns an iterator of objects (files or directories) stored under the provided prefix. 488 489 :param url: The prefix to list objects under. 490 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist. 491 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist. 492 :param include_directories: Whether to include directories in the result. When True, directories are returned alongside objects. 493 :param attribute_filter_expression: The attribute filter expression to apply to the result. 494 :param show_attributes: Whether to return attributes in the result. 495 :param patterns: PatternList for include/exclude filtering. If None, all files are included. 496 :param symlink_handling: How to handle symbolic links. Only applicable for POSIX file storage. 497 :return: An iterator of :py:class:`ObjectMetadata` objects representing the files (and optionally directories) 498 accessible under the specified URL prefix. The returned keys will always be prefixed with msc://. 499 """ 500 client, path = resolve_storage_client(url) 501 return client.list( 502 path=path, 503 start_after=start_after, 504 end_at=end_at, 505 include_directories=include_directories, 506 include_url_prefix=True, 507 attribute_filter_expression=attribute_filter_expression, 508 show_attributes=show_attributes, 509 patterns=patterns, 510 symlink_handling=symlink_handling, 511 )
512 513
[docs] 514def list_recursive( 515 url: str, 516 start_after: str | None = None, 517 end_at: str | None = None, 518 max_workers: int = 32, 519 look_ahead: int = 2, 520 patterns: PatternList | None = None, 521 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW, 522) -> Iterator[ObjectMetadata]: 523 """ 524 Lists files recursively under the specified URL. 525 526 This function retrieves the corresponding :py:class:`multistorageclient.StorageClient` 527 for the given URL and returns an iterator of files under the provided path. 528 529 :param url: The path to list objects under. 530 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist. 531 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist. 532 :param max_workers: Maximum concurrent workers for provider-level recursive listing. 533 :param look_ahead: Prefixes to buffer per worker for provider-level recursive listing. 534 :param patterns: PatternList for include/exclude filtering. If None, all files are included. 535 :param symlink_handling: How to handle symbolic links during listing. 536 :return: An iterator of :py:class:`ObjectMetadata` objects representing files accessible under the specified URL path. 537 The returned keys use the same URL-prefix behavior as :py:meth:`multistorageclient.list`. 538 """ 539 client, path = resolve_storage_client(url) 540 return client.list_recursive( 541 path=path, 542 start_after=start_after, 543 end_at=end_at, 544 max_workers=max_workers, 545 look_ahead=look_ahead, 546 include_url_prefix=True, 547 patterns=patterns, 548 symlink_handling=symlink_handling, 549 )
550 551
[docs] 552def write(url: str, body: bytes, attributes: dict[str, Any] | None = None) -> None: 553 """ 554 Writes an object to the storage provider at the specified path. 555 556 :param url: The path where the object should be written. 557 :param body: The content to write to the object. 558 """ 559 client, path = resolve_storage_client(url) 560 client.write(path=path, body=body, attributes=attributes)
561 562 578 579
[docs] 580def delete(url: str, recursive: bool = False) -> None: 581 """ 582 Deletes the specified object(s) from the storage provider. 583 584 This function retrieves the corresponding :py:class:`multistorageclient.StorageClient` 585 for the given URL and deletes the object(s) at the specified path. 586 587 :param url: The URL of the object to delete. (example: ``msc://profile/prefix/file.txt``) 588 :param recursive: Whether to delete objects in the path recursively. 589 """ 590 client, path = resolve_storage_client(url) 591 client.delete(path, recursive=recursive)
592 593
[docs] 594def info(url: str) -> ObjectMetadata: 595 """ 596 Retrieves metadata or information about an object stored at the specified path. 597 598 :param url: The URL of the object to retrieve information about. (example: ``msc://profile/prefix/file.txt``) 599 600 :return: An :py:class:`ObjectMetadata` object representing the object's metadata. 601 """ 602 client, path = resolve_storage_client(url) 603 return client.info(path)
604 605
[docs] 606def commit_metadata(url: str) -> None: 607 """ 608 Commits the metadata updates for the specified storage client profile. 609 610 :param url: The URL of the path to commit metadata for. 611 """ 612 client, path = resolve_storage_client(url) 613 client.commit_metadata(prefix=path)
614 615
[docs] 616def generate_presigned_url( 617 url: str, 618 *, 619 method: str = "GET", 620 signer_type: SignerType | None = None, 621 signer_options: dict[str, Any] | None = None, 622) -> str: 623 """ 624 Generate a pre-signed URL granting temporary access to the object at *url*. 625 626 :param url: The storage URL. (example: ``msc://profile/prefix/file.bin``) 627 :param method: The HTTP method the URL should authorise (e.g. ``"GET"``, ``"PUT"``). 628 :param signer_type: The signing backend to use. ``None`` means the provider's native signer. 629 :param signer_options: Backend-specific options forwarded to the signer. 630 :return: A pre-signed URL string. 631 """ 632 client, path = resolve_storage_client(url) 633 return client.generate_presigned_url(path, method=method, signer_type=signer_type, signer_options=signer_options)