Source code for multistorageclient.types

   1# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
   2# SPDX-License-Identifier: Apache-2.0
   3#
   4# Licensed under the Apache License, Version 2.0 (the "License");
   5# you may not use this file except in compliance with the License.
   6# You may obtain a copy of the License at
   7#
   8# http://www.apache.org/licenses/LICENSE-2.0
   9#
  10# Unless required by applicable law or agreed to in writing, software
  11# distributed under the License is distributed on an "AS IS" BASIS,
  12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13# See the License for the specific language governing permissions and
  14# limitations under the License.
  15
  16from __future__ import annotations
  17
  18import posixpath
  19from abc import ABC, abstractmethod
  20from collections.abc import Iterator, Sequence
  21from dataclasses import asdict, dataclass, field, replace
  22from datetime import UTC, datetime
  23from enum import Enum
  24from typing import IO, Any, Literal, NamedTuple, Protocol
  25
  26from dateutil.parser import parse as dateutil_parser
  27
  28MSC_PROTOCOL_NAME = "msc"
  29MSC_PROTOCOL = MSC_PROTOCOL_NAME + "://"
  30
  31TransferState = Literal["started", "completed", "failed"]
  32"""Lifecycle state of one file transfer attempt; failure includes cancellation."""
  33
  34
[docs] 35class BatchTransferCallback(Protocol): 36 """Notify the lifecycle of each file attempt in a batch transfer. 37 38 Paths identify the transfer at the emitting layer; remote paths may already 39 be resolved by a metadata provider. Calls are synchronous and may be concurrent. 40 """ 41 42 def __call__( 43 self, 44 source_path: str, 45 destination_path: str, 46 state: TransferState, 47 /, 48 ) -> None: 49 """Report a started, completed, or failed file transfer attempt.""" 50 ...
51 52 53DEFAULT_RETRY_ATTEMPTS = 3 54DEFAULT_RETRY_DELAY = 1.0 55DEFAULT_RETRY_BACKOFF_MULTIPLIER = 2.0 56 57# datetime.min is a naive datetime. 58# 59# This creates issues when doing datetime.astimezone(timezone.utc) since it assumes the local timezone for the naive datetime. 60# If the local timezone is offset behind UTC, it attempts to subtract off the offset which goes below the representable limit (i.e. an underflow). 61# A `ValueError: year 0 is out of range` is thrown as a result. 62AWARE_DATETIME_MIN = datetime.min.replace(tzinfo=UTC) 63 64
[docs] 65class SymlinkHandling(str, Enum): 66 """Controls how symbolic links are handled during listing and sync operations. 67 68 External symlinks (targets outside ``base_path``) are skipped by default in 69 ``FOLLOW`` and ``PRESERVE``. Use ``FOLLOW_STRICT`` or ``PRESERVE_STRICT`` to 70 fail instead. 71 72 - ``FOLLOW``: Dereference internal symlinks; skip external symlinks. This is 73 the default. 74 - ``FOLLOW_STRICT``: Dereference internal symlinks; raise on external symlinks. 75 - ``SKIP``: All symlinks are excluded from results. 76 - ``PRESERVE``: Surface internal symlinks as leaf entries with 77 :py:attr:`ObjectMetadata.symlink_target` populated; skip external symlinks. 78 Directory symlinks are **not** recursed into. 79 - ``PRESERVE_STRICT``: Same as ``PRESERVE`` for internal symlinks; raise on 80 external symlinks. 81 82 .. note:: 83 This option is only meaningful for POSIX file storage providers, which 84 have native symlink semantics. Cloud storage providers (S3, GCS, Azure, 85 OCI, AIS) ignore this parameter: they always list whatever is in the 86 bucket/prefix and surface MSC's symlink convention (an empty object 87 whose user metadata carries the target path) with 88 :py:attr:`ObjectMetadata.symlink_target` populated -- object storage 89 isn't a filesystem, so there is nothing to "follow" during listing. 90 """ 91 92 FOLLOW = "follow" 93 FOLLOW_STRICT = "follow_strict" 94 SKIP = "skip" 95 PRESERVE = "preserve" 96 PRESERVE_STRICT = "preserve_strict"
97 98 99# Maximum number of symlink hops allowed when resolving a symlink chain. 100MAX_SYMLINK_DEPTH = 8 101 102
[docs] 103class SignerType(str, Enum): 104 """Supported signer backends for presigned URL generation.""" 105 106 S3 = "s3" 107 CLOUDFRONT = "cloudfront" 108 AZURE = "azure"
109 110
[docs] 111@dataclass 112class Credentials: 113 """ 114 A data class representing the credentials needed to access a storage provider. 115 """ 116 117 #: The access key for authentication. 118 access_key: str 119 #: The secret key for authentication. 120 secret_key: str 121 #: An optional security token for temporary credentials. 122 token: str | None 123 #: The expiration time of the credentials in ISO 8601 format. 124 expiration: str | None 125 #: A dictionary for storing custom key-value pairs. 126 custom_fields: dict[str, Any] = field(default_factory=dict) 127
[docs] 128 def is_expired(self) -> bool: 129 """ 130 Checks if the credentials are expired based on the expiration time. 131 132 :return: ``True`` if the credentials are expired, ``False`` otherwise. 133 """ 134 expiry = dateutil_parser(self.expiration) if self.expiration else None 135 if expiry is None: 136 return False 137 return expiry <= datetime.now(tz=UTC)
138
[docs] 139 def get_custom_field(self, key: str, default: Any = None) -> Any: 140 """ 141 Retrieves a value from custom fields by its key. 142 143 :param key: The key to look up in custom fields. 144 :param default: The default value to return if the key is not found. 145 :return: The value associated with the key, or the default value if not found. 146 """ 147 return self.custom_fields.get(key, default)
148 149
[docs] 150@dataclass 151class ObjectMetadata: 152 """ 153 A data class that represents the metadata associated with an object stored in a cloud storage service. This metadata 154 includes both required and optional information about the object. 155 """ 156 157 #: Relative path of the object. 158 key: str 159 #: The size of the object in bytes. 160 content_length: int 161 #: The timestamp indicating when the object was last modified. 162 last_modified: datetime 163 type: str = "file" 164 #: The MIME type of the object. 165 content_type: str | None = field(default=None) 166 #: The entity tag (ETag) of the object. 167 etag: str | None = field(default=None) 168 #: The storage class of the object. 169 storage_class: str | None = field(default=None) 170 171 metadata: dict[str, Any] | None = field(default=None) 172 173 #: Symlink target relative to the symlink's own parent directory 174 #: (e.g. ``"../target.txt"``), or ``None`` for non-symlink entries. 175 #: See :meth:`encode_symlink_target` / :meth:`resolve_symlink_target`. 176 symlink_target: str | None = field(default=None) 177
[docs] 178 @staticmethod 179 def from_dict(data: dict) -> ObjectMetadata: 180 """ 181 Creates an ObjectMetadata instance from a dictionary (parsed from JSON). 182 """ 183 try: 184 last_modified = dateutil_parser(data["last_modified"]) 185 key = data.get("key") 186 if key is None: 187 raise ValueError("Missing required field: 'key'") 188 return ObjectMetadata( 189 key=key, 190 content_length=data["content_length"], 191 last_modified=last_modified, 192 type=data.get("type", "file"), # default to file 193 content_type=data.get("content_type"), 194 etag=data.get("etag"), 195 storage_class=data.get("storage_class"), 196 metadata=data.get("metadata"), 197 symlink_target=data.get("symlink_target"), 198 ) 199 except KeyError as e: 200 raise ValueError("Missing required field.") from e
201
[docs] 202 def replace(self, **changes: Any) -> ObjectMetadata: 203 """Return a shallow copy of this object with the given fields overridden.""" 204 return replace(self, **changes)
205
[docs] 206 def to_dict(self) -> dict: 207 data = asdict(self) 208 data["last_modified"] = self.last_modified.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") 209 return {k: v for k, v in data.items() if v is not None}
210 232
249 250
[docs] 251class CredentialsProvider(ABC): 252 """ 253 Abstract base class for providing credentials to access a storage provider. 254 """ 255
[docs] 256 @abstractmethod 257 def get_credentials(self) -> Credentials: 258 """ 259 Retrieves the current credentials. 260 261 :return: The current credentials used for authentication. 262 """
263
[docs] 264 @abstractmethod 265 def refresh_credentials(self) -> None: 266 """ 267 Refreshes the credentials if they are expired or about to expire. 268 """
269 270
[docs] 271@dataclass 272class Range: 273 """ 274 A data class that represents a byte range for read operations. 275 """ 276 277 #: The start offset in bytes. 278 offset: int 279 #: The number of bytes to read. 280 size: int
281 282
[docs] 283@dataclass 284class SuffixRange: 285 """ 286 A data class that represents the last ``size`` bytes of an object for read operations. 287 288 Reads return the whole object if it is shorter than ``size``. 289 """ 290 291 #: The number of bytes to read from the end of the object. Must be positive. 292 size: int 293 294 def __post_init__(self) -> None: 295 if self.size <= 0: 296 raise ValueError(f"SuffixRange size must be positive, got {self.size}")
297 298
[docs] 299class StorageProvider(ABC): 300 """ 301 Abstract base class for interacting with a storage provider. 302 """ 303
[docs] 304 @abstractmethod 305 def put_object( 306 self, 307 path: str, 308 body: bytes, 309 if_match: str | None = None, 310 if_none_match: str | None = None, 311 attributes: dict[str, Any] | None = None, 312 ) -> None: 313 """ 314 Uploads an object to the storage provider. 315 316 :param path: The path where the object will be stored. 317 :param body: The content of the object to store. 318 :param if_match: Optional If-Match value for conditional upload. 319 :param if_none_match: Optional If-None-Match value for conditional upload. 320 :param attributes: The attributes to add to the file. 321 """
322
[docs] 323 @abstractmethod 324 def get_object(self, path: str, byte_range: Range | None = None) -> bytes: 325 """ 326 Retrieves an object from the storage provider. 327 328 :param path: The path where the object is stored. 329 :param byte_range: Optional byte range (offset, length) to read. 330 :return: The content of the retrieved object. 331 """
332
[docs] 333 @abstractmethod 334 def copy_object(self, src_path: str, dest_path: str) -> None: 335 """ 336 Copies an object from source to destination in the storage provider. 337 338 :param src_path: The path of the source object to copy. 339 :param dest_path: The path of the destination. 340 """
341
[docs] 342 @abstractmethod 343 def delete_object(self, path: str, if_match: str | None = None) -> None: 344 """ 345 Deletes an object from the storage provider. 346 347 :param path: The path of the object to delete. 348 :param if_match: Optional if-match value to use for conditional deletion. 349 """
350
[docs] 351 @abstractmethod 352 def delete_objects(self, paths: Sequence[str]) -> None: 353 """ 354 Deletes multiple objects from the storage provider. 355 356 :param paths: A list of paths of objects to delete. 357 """
358 371
[docs] 372 @abstractmethod 373 def get_object_metadata(self, path: str, strict: bool = True) -> ObjectMetadata: 374 """ 375 Retrieves metadata or information about an object stored in the provider. 376 377 :param path: The path of the object. 378 :param strict: When ``True``, performs additional validation to determine whether the path refers to a directory. 379 380 :return: A metadata object containing the information about the object. 381 """
382
[docs] 383 @abstractmethod 384 def list_objects( 385 self, 386 path: str, 387 start_after: str | None = None, 388 end_at: str | None = None, 389 include_directories: bool = False, 390 attribute_filter_expression: str | None = None, 391 show_attributes: bool = False, 392 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW, 393 ) -> Iterator[ObjectMetadata]: 394 """ 395 Lists objects in the storage provider under the specified path. 396 397 :param path: The path to list objects under. The path must be a valid file or subdirectory path, cannot be partial or just "prefix". 398 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist. 399 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist. 400 :param include_directories: Whether to include directories in the result. When ``True``, directories are returned alongside objects. 401 :param attribute_filter_expression: The attribute filter expression to apply to the result. 402 :param show_attributes: Whether to return attributes in the result. There will be performance impact if this is True as now we need to get object metadata for each object. 403 :param symlink_handling: How to handle symbolic links during listing. 404 405 :return: An iterator over objects metadata under the specified path. 406 """
407
[docs] 408 @abstractmethod 409 def list_objects_recursive( 410 self, 411 path: str = "", 412 start_after: str | None = None, 413 end_at: str | None = None, 414 max_workers: int = 32, 415 look_ahead: int = 2, 416 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW, 417 ) -> Iterator[ObjectMetadata]: 418 """ 419 Lists files recursively in the storage provider under the specified path. 420 421 :param path: The path to list objects under. 422 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist. 423 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist. 424 :param max_workers: Maximum concurrent workers for provider-level recursive listing. 425 :param look_ahead: Prefixes to buffer per worker for provider-level recursive listing. 426 :param symlink_handling: How to handle symbolic links during listing. 427 :return: An iterator over object metadata under the specified path. 428 """
429
[docs] 430 @abstractmethod 431 def upload_file(self, remote_path: str, f: str | IO, attributes: dict[str, Any] | None = None) -> None: 432 """ 433 Uploads a file from the local file system to the storage provider. 434 435 :param remote_path: The path where the object will be stored. 436 :param f: The source file to upload. This can either be a string representing the local 437 file path, or a file-like object (e.g., an open file handle). 438 :param attributes: The attributes to add to the file if a new file is created. 439 """
440
[docs] 441 @abstractmethod 442 def download_file(self, remote_path: str, f: str | IO, metadata: ObjectMetadata | None = None) -> None: 443 """ 444 Downloads a file from the storage provider to the local file system. 445 446 :param remote_path: The path of the file to download. 447 :param f: The destination for the downloaded file. This can either be a string representing 448 the local file path where the file will be saved, or a file-like object to write the 449 downloaded content into. 450 :param metadata: Metadata about the object to download. 451 :raises FileNotFoundError: If the remote file does not exist. 452 :raises PermissionError: If the provider reports denied read access. 453 :raises RetryableError: If the provider reports a retryable transfer failure. 454 """
455
[docs] 456 @abstractmethod 457 def download_files( 458 self, 459 remote_paths: Sequence[str], 460 local_paths: Sequence[str], 461 metadata: Sequence[ObjectMetadata | None] | None = None, 462 max_workers: int = 16, 463 transfer_callback: BatchTransferCallback | None = None, 464 ) -> None: 465 """ 466 Downloads multiple files from the storage provider to the local file system. 467 468 :param remote_paths: List of remote paths of files to download. 469 :param local_paths: List of local file paths to save the downloaded files to. 470 :param metadata: Optional per-file metadata used to decide between regular and multipart download. 471 :param max_workers: Maximum number of concurrent download workers (default: 16). 472 :param transfer_callback: Optional observer called with source path, destination path, and "started", 473 "completed", or "failed" for each executing attempt. Remote paths may be resolved physical paths. 474 Calls may be concurrent; callback exceptions are logged. 475 :raises ValueError: If remote_paths and local_paths have different lengths. 476 """
477
[docs] 478 @abstractmethod 479 def upload_files( 480 self, 481 local_paths: Sequence[str], 482 remote_paths: Sequence[str], 483 attributes: Sequence[dict[str, Any] | None] | None = None, 484 max_workers: int = 16, 485 transfer_callback: BatchTransferCallback | None = None, 486 ) -> None: 487 """ 488 Uploads multiple files from the local file system to the storage provider. 489 490 :param local_paths: List of local file paths to upload. 491 :param remote_paths: List of remote paths to upload the files to. 492 :param attributes: Optional list of per-file attributes to add. When provided, must have the same length 493 as local_paths/remote_paths. Each element may be ``None`` for files that need no attributes. 494 :param max_workers: Maximum number of concurrent upload workers (default: 16). 495 :param transfer_callback: Optional observer called with source path, destination path, and "started", 496 "completed", or "failed" for each executing attempt. Remote paths may be resolved physical paths. 497 Calls may be concurrent; callback exceptions are logged. 498 :raises ValueError: If local_paths and remote_paths have different lengths. 499 :raises ValueError: If attributes is provided and has a different length than local_paths. 500 """
501
[docs] 502 @abstractmethod 503 def glob(self, pattern: str, attribute_filter_expression: str | None = None) -> list[str]: 504 """ 505 Matches and retrieves a list of object keys in the storage provider that match the specified pattern. 506 507 :param pattern: The pattern to match object keys against, supporting wildcards (e.g., ``*.txt``). 508 :param attribute_filter_expression: The attribute filter expression to apply to the result. 509 510 :return: A list of object keys that match the specified pattern. 511 """
512
[docs] 513 @abstractmethod 514 def is_file(self, path: str) -> bool: 515 """ 516 Checks whether the specified key in the storage provider points to a file (as opposed to a folder or directory). 517 518 :param path: The path to check. 519 520 :return: ``True`` if the key points to a file, ``False`` if it points to a directory or folder. 521 """
522
[docs] 523 def generate_presigned_url( 524 self, 525 path: str, 526 *, 527 method: str = "GET", 528 signer_type: SignerType | None = None, 529 signer_options: dict[str, Any] | None = None, 530 ) -> str: 531 """ 532 Generate a pre-signed URL granting temporary access to the object at *path*. 533 534 :param path: The object path within the storage provider. 535 :param method: The HTTP method the URL should authorise (e.g. ``"GET"``, ``"PUT"``). 536 :param signer_type: The signing backend to use. ``None`` means the provider's native signer. 537 :param signer_options: Backend-specific options forwarded to the signer. 538 :return: A pre-signed URL string. 539 :raises NotImplementedError: If this storage provider does not support presigned URLs. 540 """ 541 raise NotImplementedError(f"{type(self).__name__} does not support presigned URL generation.")
542 543
[docs] 544class ResolvedPathState(str, Enum): 545 """ 546 Enum representing the state of a resolved path. 547 """ 548 549 EXISTS = "exists" # File currently exists 550 DELETED = "deleted" # File existed before but has been deleted 551 UNTRACKED = "untracked" # File never existed or was never tracked
552 553
[docs] 554class ResolvedPath(NamedTuple): 555 """ 556 Result of resolving a virtual path to a physical path. 557 558 :param physical_path: The physical path in storage backend 559 :param state: The state of the path (EXISTS, DELETED, or UNTRACKED) 560 :param profile: Optional profile name for routing in CompositeStorageClient. 561 None means use current client's storage provider. 562 String means route to named child StorageClient. 563 564 State meanings: 565 - EXISTS: File currently exists in metadata 566 - DELETED: File existed before but has been deleted (soft delete) 567 - UNTRACKED: File never existed or was never tracked 568 """ 569 570 physical_path: str 571 state: ResolvedPathState 572 profile: str | None = None 573 574 @property 575 def exists(self) -> bool: 576 """Backward compatibility property: True if state is EXISTS.""" 577 return self.state == ResolvedPathState.EXISTS
578 579
[docs] 580class MetadataProvider(ABC): 581 """ 582 Abstract base class for accessing file metadata. 583 """ 584
[docs] 585 @abstractmethod 586 def list_objects( 587 self, 588 path: str, 589 start_after: str | None = None, 590 end_at: str | None = None, 591 include_directories: bool = False, 592 attribute_filter_expression: str | None = None, 593 show_attributes: bool = False, 594 ) -> Iterator[ObjectMetadata]: 595 """ 596 Lists objects in the metadata provider under the specified path. 597 598 :param path: The path to list objects under. The path must be a valid file or subdirectory path, cannot be partial or just "prefix". 599 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist. 600 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist. 601 :param include_directories: Whether to include directories in the result. When ``True``, directories are returned alongside objects. 602 :param attribute_filter_expression: The attribute filter expression to apply to the result. 603 :param show_attributes: Whether to return attributes in the result. Depending on implementation, there may be a performance impact if this is set to ``True``. 604 605 :return: An iterator over object metadata under the specified path. 606 """
607
[docs] 608 @abstractmethod 609 def get_object_metadata(self, path: str, include_pending: bool = False) -> ObjectMetadata: 610 """ 611 Retrieves metadata or information about an object or directory stored in the provider. 612 613 If the path does not match a file, implementations should check whether the path 614 represents a valid directory (i.e. files exist under the path prefix) and return 615 directory metadata accordingly. 616 617 :param path: The path of the object or directory. 618 :param include_pending: Whether to include metadata that is not yet committed. 619 620 :return: A metadata object containing the information about the object or directory. 621 :raises FileNotFoundError: If no object or directory exists at the specified path. 622 """
623
[docs] 624 @abstractmethod 625 def glob(self, pattern: str, attribute_filter_expression: str | None = None) -> list[str]: 626 """ 627 Matches and retrieves a list of object keys in the storage provider that match the specified pattern. 628 629 :param pattern: The pattern to match object keys against, supporting wildcards (e.g., ``*.txt``). 630 :param attribute_filter_expression: The attribute filter expression to apply to the result. 631 632 :return: A list of object keys that match the specified pattern. 633 """
634
[docs] 635 @abstractmethod 636 def realpath(self, logical_path: str) -> ResolvedPath: 637 """ 638 Resolves a logical path to its physical storage path. 639 640 This method checks if the object exists in the committed state and returns 641 the appropriate physical path with the current state of the path. 642 643 :param logical_path: The user-facing logical path 644 645 :return: ResolvedPath with physical_path and state: 646 - ResolvedPathState.EXISTS: File currently exists 647 - ResolvedPathState.UNTRACKED: File never existed 648 - ResolvedPathState.DELETED: File was deleted 649 If state is EXISTS, physical_path is the committed storage path. 650 Otherwise, physical_path is typically the logical_path as fallback. 651 """
652
[docs] 653 @abstractmethod 654 def generate_physical_path(self, logical_path: str, for_overwrite: bool = False) -> ResolvedPath: 655 """ 656 Generates a physical storage path for writing a new or overwritten object. 657 658 This method is used for write operations to determine where the object should 659 be physically stored. Implementations can use this to: 660 - Generate UUID-based paths for deduplication 661 - Create versioned paths (file-v1.txt, file-v2.txt) for time travel 662 - Implement path rewriting strategies 663 664 :param logical_path: The user-facing logical path 665 :param for_overwrite: When ``True``, indicates the path is for overwriting an existing object. 666 Implementations may generate unique paths for overwrites to support versioning. 667 668 :return: ResolvedPath with physical_path for writing. The exists flag indicates 669 whether the logical path currently exists in committed state (for overwrite scenarios). 670 """
671
[docs] 672 @abstractmethod 673 def add_file(self, path: str, metadata: ObjectMetadata) -> None: 674 """ 675 Add a file to be tracked by the :py:class:`MetadataProvider`. Does not have to be 676 reflected in listing until a :py:meth:`MetadataProvider.commit_updates` forces a persist. 677 This function must tolerate duplicate calls (idempotent behavior). 678 679 :param path: User-supplied virtual path 680 :param metadata: physical file metadata from StorageProvider 681 """
682
[docs] 683 @abstractmethod 684 def remove_file(self, path: str) -> None: 685 """ 686 Remove a file tracked by the :py:class:`MetadataProvider`. Does not have to be 687 reflected in listing until a :py:meth:`MetadataProvider.commit_updates` forces a persist. 688 This function must tolerate duplicate calls (idempotent behavior). 689 690 :param path: User-supplied virtual path 691 """
692
[docs] 693 @abstractmethod 694 def commit_updates(self) -> None: 695 """ 696 Commit any newly adding files, used in conjunction with :py:meth:`MetadataProvider.add_file`. 697 :py:class:`MetadataProvider` will persistently record any metadata changes. 698 """
699
[docs] 700 @abstractmethod 701 def is_writable(self) -> bool: 702 """ 703 Returns ``True`` if the :py:class:`MetadataProvider` supports writes else ``False``. 704 """
705
[docs] 706 @abstractmethod 707 def allow_overwrites(self) -> bool: 708 """ 709 Returns ``True`` if the :py:class:`MetadataProvider` allows overwriting existing files else ``False``. 710 When ``True``, :py:meth:`add_file` will not raise an error if the file already exists. 711 """
712
[docs] 713 @abstractmethod 714 def should_use_soft_delete(self) -> bool: 715 """ 716 Returns ``True`` if the :py:class:`MetadataProvider` should use soft-delete behavior else ``False``. 717 718 When ``True``, delete operations will only mark files as deleted in metadata without removing 719 the physical data from storage. The file will return :py:class:`ResolvedPathState.DELETED` state 720 when queried and will not appear in listings. 721 722 When ``False``, delete operations will remove both the metadata and the physical file from storage 723 (hard delete). 724 """
725 726
[docs] 727@dataclass 728class StorageProviderConfig: 729 """ 730 A data class that represents the configuration needed to initialize a storage provider. 731 """ 732 733 #: The name or type of the storage provider (e.g., ``s3``, ``gcs``, ``oci``, ``azure``). 734 type: str 735 #: Additional options required to configure the storage provider (e.g., endpoint URLs, region, etc.). 736 options: dict[str, Any] | None = None
737 738
[docs] 739@dataclass 740class StorageBackend: 741 """ 742 Represents configuration for a single storage backend. 743 """ 744 745 storage_provider_config: StorageProviderConfig 746 credentials_provider: CredentialsProvider | None = None 747 replicas: list[Replica] = field(default_factory=list)
748 749
[docs] 750class ProviderBundle(ABC): 751 """ 752 Abstract base class that serves as a container for various providers (storage, credentials, and metadata) 753 that interact with a storage service. The :py:class:`ProviderBundle` abstracts access to these providers, allowing for 754 flexible implementations of cloud storage solutions. 755 """ 756 757 @property 758 @abstractmethod 759 def storage_provider_config(self) -> StorageProviderConfig: 760 """ 761 :return: The configuration for the storage provider, which includes the provider 762 name/type and additional options. 763 """ 764 765 @property 766 @abstractmethod 767 def credentials_provider(self) -> CredentialsProvider | None: 768 """ 769 :return: The credentials provider responsible for managing authentication credentials 770 required to access the storage service. 771 """ 772 773 @property 774 @abstractmethod 775 def metadata_provider(self) -> MetadataProvider | None: 776 """ 777 :return: The metadata provider responsible for retrieving metadata about objects in the storage service. 778 """ 779 780 @property 781 @abstractmethod 782 def replicas(self) -> list[Replica]: 783 """ 784 :return: The replicas configuration for this provider bundle, if any. 785 """
786 787
[docs] 788class ProviderBundleV2(ABC): 789 """ 790 Abstract base class that serves as a container for various providers (storage, credentials, and metadata) 791 that interact with one or multiple storage service. The :py:class:`ProviderBundleV2` abstracts access to these providers, allowing for 792 flexible implementations of cloud storage solutions. 793 794 """ 795 796 @property 797 @abstractmethod 798 def storage_backends(self) -> dict[str, StorageBackend]: 799 """ 800 :return: Mapping of storage backend name -> StorageBackend. Must have at least one backend. 801 """ 802 803 @property 804 @abstractmethod 805 def metadata_provider(self) -> MetadataProvider | None: 806 """ 807 :return: The metadata provider responsible for retrieving metadata about objects in the storage service. If there are multiple backends, this is required. 808 """
809 810
[docs] 811@dataclass 812class RetryConfig: 813 """ 814 A data class that represents the configuration for retry strategy. 815 """ 816 817 #: The number of attempts before giving up. Must be at least 1. 818 attempts: int = DEFAULT_RETRY_ATTEMPTS 819 #: The base delay (in seconds) for exponential backoff. Must be a non-negative value. 820 delay: float = DEFAULT_RETRY_DELAY 821 #: The backoff multiplier for exponential backoff. Must be at least 1.0. 822 backoff_multiplier: float = DEFAULT_RETRY_BACKOFF_MULTIPLIER 823 824 def __post_init__(self) -> None: 825 if self.attempts < 1: 826 raise ValueError("Attempts must be at least 1.") 827 if self.delay < 0: 828 raise ValueError("Delay must be a non-negative number.") 829 if self.backoff_multiplier < 1.0: 830 raise ValueError("Backoff multiplier must be at least 1.0.")
831 832
[docs] 833class RetryableError(Exception): 834 """ 835 Exception raised for errors that should trigger a retry. 836 """
837 838
[docs] 839@dataclass(frozen=True) 840class BatchTransferFailure: 841 """ 842 A failed item from a batch upload or download operation. 843 """ 844 845 #: The item index in the batch request that failed. 846 index: int 847 #: The source path for the failed transfer. 848 source_path: str 849 #: The destination path for the failed transfer. 850 destination_path: str 851 #: The underlying exception raised for this item. 852 error: Exception
853 854
[docs] 855class BatchTransferError(Exception): 856 """ 857 Exception raised when one or more items fail in a batch transfer operation. 858 """ 859 860 def __init__(self, failures: Sequence[BatchTransferFailure]): 861 if not failures: 862 raise ValueError("BatchTransferError requires at least one failure.") 863 self.failures = list(failures) 864 super().__init__(self._format_message()) 865 866 def _format_message(self) -> str: 867 details = ", ".join( 868 f"index {failure.index} ({type(failure.error).__name__}: {failure.error})" for failure in self.failures[:3] 869 ) 870 if len(self.failures) > 3: 871 details += f", and {len(self.failures) - 3} more" 872 return f"{len(self.failures)} batch transfer item(s) failed: {details}"
873 874
[docs] 875class PreconditionFailedError(Exception): 876 """ 877 Exception raised when a precondition fails. e.g. if-match, if-none-match, etc. 878 """
879 880
[docs] 881class NotModifiedError(Exception): 882 """ 883 Raised when a conditional operation fails because the resource has not been modified. 884 885 This typically occurs when using if-none-match with a specific generation/etag 886 and the resource's current generation/etag matches the specified one. 887 """
888 889
[docs] 890class SourceVersionCheckMode(Enum): 891 """ 892 Enum for controlling source version checking behavior. 893 """ 894 895 INHERIT = "inherit" # Inherit from configuration (cache config) 896 ENABLE = "enable" # Always check source version 897 DISABLE = "disable" # Never check source version
898 899
[docs] 900@dataclass 901class Replica: 902 """ 903 A storage tier that can serve reads and optionally receive writes. 904 """ 905 906 replica_profile: str 907 read_priority: int 908 read_only: bool = False
909 910
[docs] 911class AutoCommitConfig: 912 """ 913 A data class that represents the configuration for auto commit. 914 """ 915 916 interval_minutes: float | None # The interval in minutes for auto commit. 917 at_exit: bool = False # if True, commit on program exit 918 919 def __init__(self, interval_minutes: float | None = None, at_exit: bool = False) -> None: 920 self.interval_minutes = interval_minutes 921 self.at_exit = at_exit
922 923
[docs] 924class ExecutionMode(Enum): 925 """ 926 Enum for controlling execution mode in sync operations. 927 """ 928 929 LOCAL = "local" 930 RAY = "ray"
931 932
[docs] 933class PatternType(Enum): 934 """ 935 Type of pattern operation for include/exclude filtering. 936 """ 937 938 INCLUDE = "include" 939 EXCLUDE = "exclude"
940 941 942# Type alias for pattern matching 943PatternList = list[tuple[PatternType, str]] 944 945
[docs] 946@dataclass 947class DryrunResult: 948 """ 949 Holds references to JSONL files produced by a dryrun sync operation. 950 951 Each file contains one JSON object per line, matching the :py:class:`ObjectMetadata` 952 serialization format (see :py:meth:`ObjectMetadata.to_dict` / :py:meth:`ObjectMetadata.from_dict`). 953 954 The caller is responsible for cleaning up the files when they are no longer needed. 955 """ 956 957 #: Path to a JSONL file listing source objects that would be added to the target. 958 files_to_add: str 959 #: Path to a JSONL file listing target objects that would be deleted. 960 files_to_delete: str
961 962
[docs] 963@dataclass 964class SyncResult: 965 """ 966 A data class that represents the summary of a sync operation. 967 """ 968 969 #: The total number of work units tracked for progress (including files from both source and target after filtering). Each work unit represents an ADD or DELETE operation. 970 total_work_units: int = 0 971 #: The total number of files processed to the target. 972 total_files_added: int = 0 973 #: The total number of files deleted from the target. 974 total_files_deleted: int = 0 975 #: The total number of bytes transferred to the target. 976 total_bytes_added: int = 0 977 #: The total number of bytes deleted from the target. 978 total_bytes_deleted: int = 0 979 #: The total time taken to process the sync operation. 980 total_time_seconds: float = 0.0 981 #: Dryrun details with paths to JSONL files. ``None`` for normal (non-dryrun) sync operations. 982 dryrun: DryrunResult | None = None 983 984 def __str__(self) -> str: 985 header = "Sync dryrun statistics:" if self.dryrun else "Sync statistics:" 986 lines = ( 987 f"{header}\n" 988 f" Work units: {self.total_work_units}\n" 989 f" Files added: {self.total_files_added}\n" 990 f" Files deleted: {self.total_files_deleted}\n" 991 f" Bytes added: {self.total_bytes_added}\n" 992 f" Bytes deleted: {self.total_bytes_deleted}\n" 993 f" Time elapsed: {self.total_time_seconds:.2f}s" 994 ) 995 if self.dryrun: 996 lines += f"\n Files to add: {self.dryrun.files_to_add}\n Files to delete: {self.dryrun.files_to_delete}" 997 return lines
998 999
[docs] 1000class SyncError(RuntimeError): 1001 """ 1002 Exception raised when errors occur during a sync operation. 1003 1004 This exception includes the partial SyncResult showing what was accomplished 1005 before the error occurred, allowing users to understand the state of the sync. 1006 1007 :param message: The error message describing what went wrong. 1008 :param sync_result: The partial SyncResult with statistics from the failed sync operation. 1009 """ 1010 1011 def __init__(self, message: str, sync_result: SyncResult): 1012 super().__init__(message) 1013 self.sync_result = sync_result 1014 1015 def __str__(self) -> str: 1016 sync_stats = str(self.sync_result).replace("Sync statistics:", "Partial sync statistics:") 1017 return f"{super().__str__()}\n\n{sync_stats}"