1# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2# SPDX-License-Identifier: Apache-2.0
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16from __future__ import annotations
17
18import posixpath
19from abc import ABC, abstractmethod
20from collections.abc import Iterator, Sequence
21from dataclasses import asdict, dataclass, field, replace
22from datetime import UTC, datetime
23from enum import Enum
24from typing import IO, Any, Literal, NamedTuple, Protocol
25
26from dateutil.parser import parse as dateutil_parser
27
28MSC_PROTOCOL_NAME = "msc"
29MSC_PROTOCOL = MSC_PROTOCOL_NAME + "://"
30
31TransferState = Literal["started", "completed", "failed"]
32"""Lifecycle state of one file transfer attempt; failure includes cancellation."""
33
34
[docs]
35class BatchTransferCallback(Protocol):
36 """Notify the lifecycle of each file attempt in a batch transfer.
37
38 Paths identify the transfer at the emitting layer; remote paths may already
39 be resolved by a metadata provider. Calls are synchronous and may be concurrent.
40 """
41
42 def __call__(
43 self,
44 source_path: str,
45 destination_path: str,
46 state: TransferState,
47 /,
48 ) -> None:
49 """Report a started, completed, or failed file transfer attempt."""
50 ...
51
52
53DEFAULT_RETRY_ATTEMPTS = 3
54DEFAULT_RETRY_DELAY = 1.0
55DEFAULT_RETRY_BACKOFF_MULTIPLIER = 2.0
56
57# datetime.min is a naive datetime.
58#
59# This creates issues when doing datetime.astimezone(timezone.utc) since it assumes the local timezone for the naive datetime.
60# If the local timezone is offset behind UTC, it attempts to subtract off the offset which goes below the representable limit (i.e. an underflow).
61# A `ValueError: year 0 is out of range` is thrown as a result.
62AWARE_DATETIME_MIN = datetime.min.replace(tzinfo=UTC)
63
64
[docs]
65class SymlinkHandling(str, Enum):
66 """Controls how symbolic links are handled during listing and sync operations.
67
68 External symlinks (targets outside ``base_path``) are skipped by default in
69 ``FOLLOW`` and ``PRESERVE``. Use ``FOLLOW_STRICT`` or ``PRESERVE_STRICT`` to
70 fail instead.
71
72 - ``FOLLOW``: Dereference internal symlinks; skip external symlinks. This is
73 the default.
74 - ``FOLLOW_STRICT``: Dereference internal symlinks; raise on external symlinks.
75 - ``SKIP``: All symlinks are excluded from results.
76 - ``PRESERVE``: Surface internal symlinks as leaf entries with
77 :py:attr:`ObjectMetadata.symlink_target` populated; skip external symlinks.
78 Directory symlinks are **not** recursed into.
79 - ``PRESERVE_STRICT``: Same as ``PRESERVE`` for internal symlinks; raise on
80 external symlinks.
81
82 .. note::
83 This option is only meaningful for POSIX file storage providers, which
84 have native symlink semantics. Cloud storage providers (S3, GCS, Azure,
85 OCI, AIS) ignore this parameter: they always list whatever is in the
86 bucket/prefix and surface MSC's symlink convention (an empty object
87 whose user metadata carries the target path) with
88 :py:attr:`ObjectMetadata.symlink_target` populated -- object storage
89 isn't a filesystem, so there is nothing to "follow" during listing.
90 """
91
92 FOLLOW = "follow"
93 FOLLOW_STRICT = "follow_strict"
94 SKIP = "skip"
95 PRESERVE = "preserve"
96 PRESERVE_STRICT = "preserve_strict"
97
98
99# Maximum number of symlink hops allowed when resolving a symlink chain.
100MAX_SYMLINK_DEPTH = 8
101
102
[docs]
103class SignerType(str, Enum):
104 """Supported signer backends for presigned URL generation."""
105
106 S3 = "s3"
107 CLOUDFRONT = "cloudfront"
108 AZURE = "azure"
109
110
[docs]
111@dataclass
112class Credentials:
113 """
114 A data class representing the credentials needed to access a storage provider.
115 """
116
117 #: The access key for authentication.
118 access_key: str
119 #: The secret key for authentication.
120 secret_key: str
121 #: An optional security token for temporary credentials.
122 token: str | None
123 #: The expiration time of the credentials in ISO 8601 format.
124 expiration: str | None
125 #: A dictionary for storing custom key-value pairs.
126 custom_fields: dict[str, Any] = field(default_factory=dict)
127
[docs]
128 def is_expired(self) -> bool:
129 """
130 Checks if the credentials are expired based on the expiration time.
131
132 :return: ``True`` if the credentials are expired, ``False`` otherwise.
133 """
134 expiry = dateutil_parser(self.expiration) if self.expiration else None
135 if expiry is None:
136 return False
137 return expiry <= datetime.now(tz=UTC)
138
[docs]
139 def get_custom_field(self, key: str, default: Any = None) -> Any:
140 """
141 Retrieves a value from custom fields by its key.
142
143 :param key: The key to look up in custom fields.
144 :param default: The default value to return if the key is not found.
145 :return: The value associated with the key, or the default value if not found.
146 """
147 return self.custom_fields.get(key, default)
148
149
249
250
[docs]
251class CredentialsProvider(ABC):
252 """
253 Abstract base class for providing credentials to access a storage provider.
254 """
255
[docs]
256 @abstractmethod
257 def get_credentials(self) -> Credentials:
258 """
259 Retrieves the current credentials.
260
261 :return: The current credentials used for authentication.
262 """
263
[docs]
264 @abstractmethod
265 def refresh_credentials(self) -> None:
266 """
267 Refreshes the credentials if they are expired or about to expire.
268 """
269
270
[docs]
271@dataclass
272class Range:
273 """
274 A data class that represents a byte range for read operations.
275 """
276
277 #: The start offset in bytes.
278 offset: int
279 #: The number of bytes to read.
280 size: int
281
282
[docs]
283@dataclass
284class SuffixRange:
285 """
286 A data class that represents the last ``size`` bytes of an object for read operations.
287
288 Reads return the whole object if it is shorter than ``size``.
289 """
290
291 #: The number of bytes to read from the end of the object. Must be positive.
292 size: int
293
294 def __post_init__(self) -> None:
295 if self.size <= 0:
296 raise ValueError(f"SuffixRange size must be positive, got {self.size}")
297
298
[docs]
299class StorageProvider(ABC):
300 """
301 Abstract base class for interacting with a storage provider.
302 """
303
[docs]
304 @abstractmethod
305 def put_object(
306 self,
307 path: str,
308 body: bytes,
309 if_match: str | None = None,
310 if_none_match: str | None = None,
311 attributes: dict[str, Any] | None = None,
312 ) -> None:
313 """
314 Uploads an object to the storage provider.
315
316 :param path: The path where the object will be stored.
317 :param body: The content of the object to store.
318 :param if_match: Optional If-Match value for conditional upload.
319 :param if_none_match: Optional If-None-Match value for conditional upload.
320 :param attributes: The attributes to add to the file.
321 """
322
[docs]
323 @abstractmethod
324 def get_object(self, path: str, byte_range: Range | None = None) -> bytes:
325 """
326 Retrieves an object from the storage provider.
327
328 :param path: The path where the object is stored.
329 :param byte_range: Optional byte range (offset, length) to read.
330 :return: The content of the retrieved object.
331 """
332
[docs]
333 @abstractmethod
334 def copy_object(self, src_path: str, dest_path: str) -> None:
335 """
336 Copies an object from source to destination in the storage provider.
337
338 :param src_path: The path of the source object to copy.
339 :param dest_path: The path of the destination.
340 """
341
[docs]
342 @abstractmethod
343 def delete_object(self, path: str, if_match: str | None = None) -> None:
344 """
345 Deletes an object from the storage provider.
346
347 :param path: The path of the object to delete.
348 :param if_match: Optional if-match value to use for conditional deletion.
349 """
350
[docs]
351 @abstractmethod
352 def delete_objects(self, paths: Sequence[str]) -> None:
353 """
354 Deletes multiple objects from the storage provider.
355
356 :param paths: A list of paths of objects to delete.
357 """
358
[docs]
359 @abstractmethod
360 def make_symlink(self, path: str, target: str) -> None:
361 """
362 Creates a symbolic link at ``path`` pointing to ``target``.
363
364 On POSIX backends this creates a native OS symlink with a relative target path.
365 On object-store backends this creates a zero-byte marker object with the target
366 stored in user metadata (``msc-symlink-target``).
367
368 :param path: The path where the symlink will be created.
369 :param target: The logical key that the symlink points to.
370 """
371
382
[docs]
383 @abstractmethod
384 def list_objects(
385 self,
386 path: str,
387 start_after: str | None = None,
388 end_at: str | None = None,
389 include_directories: bool = False,
390 attribute_filter_expression: str | None = None,
391 show_attributes: bool = False,
392 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
393 ) -> Iterator[ObjectMetadata]:
394 """
395 Lists objects in the storage provider under the specified path.
396
397 :param path: The path to list objects under. The path must be a valid file or subdirectory path, cannot be partial or just "prefix".
398 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist.
399 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist.
400 :param include_directories: Whether to include directories in the result. When ``True``, directories are returned alongside objects.
401 :param attribute_filter_expression: The attribute filter expression to apply to the result.
402 :param show_attributes: Whether to return attributes in the result. There will be performance impact if this is True as now we need to get object metadata for each object.
403 :param symlink_handling: How to handle symbolic links during listing.
404
405 :return: An iterator over objects metadata under the specified path.
406 """
407
[docs]
408 @abstractmethod
409 def list_objects_recursive(
410 self,
411 path: str = "",
412 start_after: str | None = None,
413 end_at: str | None = None,
414 max_workers: int = 32,
415 look_ahead: int = 2,
416 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
417 ) -> Iterator[ObjectMetadata]:
418 """
419 Lists files recursively in the storage provider under the specified path.
420
421 :param path: The path to list objects under.
422 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist.
423 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist.
424 :param max_workers: Maximum concurrent workers for provider-level recursive listing.
425 :param look_ahead: Prefixes to buffer per worker for provider-level recursive listing.
426 :param symlink_handling: How to handle symbolic links during listing.
427 :return: An iterator over object metadata under the specified path.
428 """
429
[docs]
430 @abstractmethod
431 def upload_file(self, remote_path: str, f: str | IO, attributes: dict[str, Any] | None = None) -> None:
432 """
433 Uploads a file from the local file system to the storage provider.
434
435 :param remote_path: The path where the object will be stored.
436 :param f: The source file to upload. This can either be a string representing the local
437 file path, or a file-like object (e.g., an open file handle).
438 :param attributes: The attributes to add to the file if a new file is created.
439 """
440
[docs]
441 @abstractmethod
442 def download_file(self, remote_path: str, f: str | IO, metadata: ObjectMetadata | None = None) -> None:
443 """
444 Downloads a file from the storage provider to the local file system.
445
446 :param remote_path: The path of the file to download.
447 :param f: The destination for the downloaded file. This can either be a string representing
448 the local file path where the file will be saved, or a file-like object to write the
449 downloaded content into.
450 :param metadata: Metadata about the object to download.
451 :raises FileNotFoundError: If the remote file does not exist.
452 :raises PermissionError: If the provider reports denied read access.
453 :raises RetryableError: If the provider reports a retryable transfer failure.
454 """
455
[docs]
456 @abstractmethod
457 def download_files(
458 self,
459 remote_paths: Sequence[str],
460 local_paths: Sequence[str],
461 metadata: Sequence[ObjectMetadata | None] | None = None,
462 max_workers: int = 16,
463 transfer_callback: BatchTransferCallback | None = None,
464 ) -> None:
465 """
466 Downloads multiple files from the storage provider to the local file system.
467
468 :param remote_paths: List of remote paths of files to download.
469 :param local_paths: List of local file paths to save the downloaded files to.
470 :param metadata: Optional per-file metadata used to decide between regular and multipart download.
471 :param max_workers: Maximum number of concurrent download workers (default: 16).
472 :param transfer_callback: Optional observer called with source path, destination path, and "started",
473 "completed", or "failed" for each executing attempt. Remote paths may be resolved physical paths.
474 Calls may be concurrent; callback exceptions are logged.
475 :raises ValueError: If remote_paths and local_paths have different lengths.
476 """
477
[docs]
478 @abstractmethod
479 def upload_files(
480 self,
481 local_paths: Sequence[str],
482 remote_paths: Sequence[str],
483 attributes: Sequence[dict[str, Any] | None] | None = None,
484 max_workers: int = 16,
485 transfer_callback: BatchTransferCallback | None = None,
486 ) -> None:
487 """
488 Uploads multiple files from the local file system to the storage provider.
489
490 :param local_paths: List of local file paths to upload.
491 :param remote_paths: List of remote paths to upload the files to.
492 :param attributes: Optional list of per-file attributes to add. When provided, must have the same length
493 as local_paths/remote_paths. Each element may be ``None`` for files that need no attributes.
494 :param max_workers: Maximum number of concurrent upload workers (default: 16).
495 :param transfer_callback: Optional observer called with source path, destination path, and "started",
496 "completed", or "failed" for each executing attempt. Remote paths may be resolved physical paths.
497 Calls may be concurrent; callback exceptions are logged.
498 :raises ValueError: If local_paths and remote_paths have different lengths.
499 :raises ValueError: If attributes is provided and has a different length than local_paths.
500 """
501
[docs]
502 @abstractmethod
503 def glob(self, pattern: str, attribute_filter_expression: str | None = None) -> list[str]:
504 """
505 Matches and retrieves a list of object keys in the storage provider that match the specified pattern.
506
507 :param pattern: The pattern to match object keys against, supporting wildcards (e.g., ``*.txt``).
508 :param attribute_filter_expression: The attribute filter expression to apply to the result.
509
510 :return: A list of object keys that match the specified pattern.
511 """
512
[docs]
513 @abstractmethod
514 def is_file(self, path: str) -> bool:
515 """
516 Checks whether the specified key in the storage provider points to a file (as opposed to a folder or directory).
517
518 :param path: The path to check.
519
520 :return: ``True`` if the key points to a file, ``False`` if it points to a directory or folder.
521 """
522
[docs]
523 def generate_presigned_url(
524 self,
525 path: str,
526 *,
527 method: str = "GET",
528 signer_type: SignerType | None = None,
529 signer_options: dict[str, Any] | None = None,
530 ) -> str:
531 """
532 Generate a pre-signed URL granting temporary access to the object at *path*.
533
534 :param path: The object path within the storage provider.
535 :param method: The HTTP method the URL should authorise (e.g. ``"GET"``, ``"PUT"``).
536 :param signer_type: The signing backend to use. ``None`` means the provider's native signer.
537 :param signer_options: Backend-specific options forwarded to the signer.
538 :return: A pre-signed URL string.
539 :raises NotImplementedError: If this storage provider does not support presigned URLs.
540 """
541 raise NotImplementedError(f"{type(self).__name__} does not support presigned URL generation.")
542
543
[docs]
544class ResolvedPathState(str, Enum):
545 """
546 Enum representing the state of a resolved path.
547 """
548
549 EXISTS = "exists" # File currently exists
550 DELETED = "deleted" # File existed before but has been deleted
551 UNTRACKED = "untracked" # File never existed or was never tracked
552
553
[docs]
554class ResolvedPath(NamedTuple):
555 """
556 Result of resolving a virtual path to a physical path.
557
558 :param physical_path: The physical path in storage backend
559 :param state: The state of the path (EXISTS, DELETED, or UNTRACKED)
560 :param profile: Optional profile name for routing in CompositeStorageClient.
561 None means use current client's storage provider.
562 String means route to named child StorageClient.
563
564 State meanings:
565 - EXISTS: File currently exists in metadata
566 - DELETED: File existed before but has been deleted (soft delete)
567 - UNTRACKED: File never existed or was never tracked
568 """
569
570 physical_path: str
571 state: ResolvedPathState
572 profile: str | None = None
573
574 @property
575 def exists(self) -> bool:
576 """Backward compatibility property: True if state is EXISTS."""
577 return self.state == ResolvedPathState.EXISTS
578
579
725
726
[docs]
727@dataclass
728class StorageProviderConfig:
729 """
730 A data class that represents the configuration needed to initialize a storage provider.
731 """
732
733 #: The name or type of the storage provider (e.g., ``s3``, ``gcs``, ``oci``, ``azure``).
734 type: str
735 #: Additional options required to configure the storage provider (e.g., endpoint URLs, region, etc.).
736 options: dict[str, Any] | None = None
737
738
[docs]
739@dataclass
740class StorageBackend:
741 """
742 Represents configuration for a single storage backend.
743 """
744
745 storage_provider_config: StorageProviderConfig
746 credentials_provider: CredentialsProvider | None = None
747 replicas: list[Replica] = field(default_factory=list)
748
749
[docs]
750class ProviderBundle(ABC):
751 """
752 Abstract base class that serves as a container for various providers (storage, credentials, and metadata)
753 that interact with a storage service. The :py:class:`ProviderBundle` abstracts access to these providers, allowing for
754 flexible implementations of cloud storage solutions.
755 """
756
757 @property
758 @abstractmethod
759 def storage_provider_config(self) -> StorageProviderConfig:
760 """
761 :return: The configuration for the storage provider, which includes the provider
762 name/type and additional options.
763 """
764
765 @property
766 @abstractmethod
767 def credentials_provider(self) -> CredentialsProvider | None:
768 """
769 :return: The credentials provider responsible for managing authentication credentials
770 required to access the storage service.
771 """
772
773 @property
774 @abstractmethod
775 def metadata_provider(self) -> MetadataProvider | None:
776 """
777 :return: The metadata provider responsible for retrieving metadata about objects in the storage service.
778 """
779
780 @property
781 @abstractmethod
782 def replicas(self) -> list[Replica]:
783 """
784 :return: The replicas configuration for this provider bundle, if any.
785 """
786
787
[docs]
788class ProviderBundleV2(ABC):
789 """
790 Abstract base class that serves as a container for various providers (storage, credentials, and metadata)
791 that interact with one or multiple storage service. The :py:class:`ProviderBundleV2` abstracts access to these providers, allowing for
792 flexible implementations of cloud storage solutions.
793
794 """
795
796 @property
797 @abstractmethod
798 def storage_backends(self) -> dict[str, StorageBackend]:
799 """
800 :return: Mapping of storage backend name -> StorageBackend. Must have at least one backend.
801 """
802
803 @property
804 @abstractmethod
805 def metadata_provider(self) -> MetadataProvider | None:
806 """
807 :return: The metadata provider responsible for retrieving metadata about objects in the storage service. If there are multiple backends, this is required.
808 """
809
810
[docs]
811@dataclass
812class RetryConfig:
813 """
814 A data class that represents the configuration for retry strategy.
815 """
816
817 #: The number of attempts before giving up. Must be at least 1.
818 attempts: int = DEFAULT_RETRY_ATTEMPTS
819 #: The base delay (in seconds) for exponential backoff. Must be a non-negative value.
820 delay: float = DEFAULT_RETRY_DELAY
821 #: The backoff multiplier for exponential backoff. Must be at least 1.0.
822 backoff_multiplier: float = DEFAULT_RETRY_BACKOFF_MULTIPLIER
823
824 def __post_init__(self) -> None:
825 if self.attempts < 1:
826 raise ValueError("Attempts must be at least 1.")
827 if self.delay < 0:
828 raise ValueError("Delay must be a non-negative number.")
829 if self.backoff_multiplier < 1.0:
830 raise ValueError("Backoff multiplier must be at least 1.0.")
831
832
[docs]
833class RetryableError(Exception):
834 """
835 Exception raised for errors that should trigger a retry.
836 """
837
838
[docs]
839@dataclass(frozen=True)
840class BatchTransferFailure:
841 """
842 A failed item from a batch upload or download operation.
843 """
844
845 #: The item index in the batch request that failed.
846 index: int
847 #: The source path for the failed transfer.
848 source_path: str
849 #: The destination path for the failed transfer.
850 destination_path: str
851 #: The underlying exception raised for this item.
852 error: Exception
853
854
[docs]
855class BatchTransferError(Exception):
856 """
857 Exception raised when one or more items fail in a batch transfer operation.
858 """
859
860 def __init__(self, failures: Sequence[BatchTransferFailure]):
861 if not failures:
862 raise ValueError("BatchTransferError requires at least one failure.")
863 self.failures = list(failures)
864 super().__init__(self._format_message())
865
866 def _format_message(self) -> str:
867 details = ", ".join(
868 f"index {failure.index} ({type(failure.error).__name__}: {failure.error})" for failure in self.failures[:3]
869 )
870 if len(self.failures) > 3:
871 details += f", and {len(self.failures) - 3} more"
872 return f"{len(self.failures)} batch transfer item(s) failed: {details}"
873
874
[docs]
875class PreconditionFailedError(Exception):
876 """
877 Exception raised when a precondition fails. e.g. if-match, if-none-match, etc.
878 """
879
880
[docs]
881class NotModifiedError(Exception):
882 """
883 Raised when a conditional operation fails because the resource has not been modified.
884
885 This typically occurs when using if-none-match with a specific generation/etag
886 and the resource's current generation/etag matches the specified one.
887 """
888
889
[docs]
890class SourceVersionCheckMode(Enum):
891 """
892 Enum for controlling source version checking behavior.
893 """
894
895 INHERIT = "inherit" # Inherit from configuration (cache config)
896 ENABLE = "enable" # Always check source version
897 DISABLE = "disable" # Never check source version
898
899
[docs]
900@dataclass
901class Replica:
902 """
903 A storage tier that can serve reads and optionally receive writes.
904 """
905
906 replica_profile: str
907 read_priority: int
908 read_only: bool = False
909
910
[docs]
911class AutoCommitConfig:
912 """
913 A data class that represents the configuration for auto commit.
914 """
915
916 interval_minutes: float | None # The interval in minutes for auto commit.
917 at_exit: bool = False # if True, commit on program exit
918
919 def __init__(self, interval_minutes: float | None = None, at_exit: bool = False) -> None:
920 self.interval_minutes = interval_minutes
921 self.at_exit = at_exit
922
923
[docs]
924class ExecutionMode(Enum):
925 """
926 Enum for controlling execution mode in sync operations.
927 """
928
929 LOCAL = "local"
930 RAY = "ray"
931
932
[docs]
933class PatternType(Enum):
934 """
935 Type of pattern operation for include/exclude filtering.
936 """
937
938 INCLUDE = "include"
939 EXCLUDE = "exclude"
940
941
942# Type alias for pattern matching
943PatternList = list[tuple[PatternType, str]]
944
945
[docs]
946@dataclass
947class DryrunResult:
948 """
949 Holds references to JSONL files produced by a dryrun sync operation.
950
951 Each file contains one JSON object per line, matching the :py:class:`ObjectMetadata`
952 serialization format (see :py:meth:`ObjectMetadata.to_dict` / :py:meth:`ObjectMetadata.from_dict`).
953
954 The caller is responsible for cleaning up the files when they are no longer needed.
955 """
956
957 #: Path to a JSONL file listing source objects that would be added to the target.
958 files_to_add: str
959 #: Path to a JSONL file listing target objects that would be deleted.
960 files_to_delete: str
961
962
[docs]
963@dataclass
964class SyncResult:
965 """
966 A data class that represents the summary of a sync operation.
967 """
968
969 #: The total number of work units tracked for progress (including files from both source and target after filtering). Each work unit represents an ADD or DELETE operation.
970 total_work_units: int = 0
971 #: The total number of files processed to the target.
972 total_files_added: int = 0
973 #: The total number of files deleted from the target.
974 total_files_deleted: int = 0
975 #: The total number of bytes transferred to the target.
976 total_bytes_added: int = 0
977 #: The total number of bytes deleted from the target.
978 total_bytes_deleted: int = 0
979 #: The total time taken to process the sync operation.
980 total_time_seconds: float = 0.0
981 #: Dryrun details with paths to JSONL files. ``None`` for normal (non-dryrun) sync operations.
982 dryrun: DryrunResult | None = None
983
984 def __str__(self) -> str:
985 header = "Sync dryrun statistics:" if self.dryrun else "Sync statistics:"
986 lines = (
987 f"{header}\n"
988 f" Work units: {self.total_work_units}\n"
989 f" Files added: {self.total_files_added}\n"
990 f" Files deleted: {self.total_files_deleted}\n"
991 f" Bytes added: {self.total_bytes_added}\n"
992 f" Bytes deleted: {self.total_bytes_deleted}\n"
993 f" Time elapsed: {self.total_time_seconds:.2f}s"
994 )
995 if self.dryrun:
996 lines += f"\n Files to add: {self.dryrun.files_to_add}\n Files to delete: {self.dryrun.files_to_delete}"
997 return lines
998
999
[docs]
1000class SyncError(RuntimeError):
1001 """
1002 Exception raised when errors occur during a sync operation.
1003
1004 This exception includes the partial SyncResult showing what was accomplished
1005 before the error occurred, allowing users to understand the state of the sync.
1006
1007 :param message: The error message describing what went wrong.
1008 :param sync_result: The partial SyncResult with statistics from the failed sync operation.
1009 """
1010
1011 def __init__(self, message: str, sync_result: SyncResult):
1012 super().__init__(message)
1013 self.sync_result = sync_result
1014
1015 def __str__(self) -> str:
1016 sync_stats = str(self.sync_result).replace("Sync statistics:", "Partial sync statistics:")
1017 return f"{super().__str__()}\n\n{sync_stats}"