1# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2# SPDX-License-Identifier: Apache-2.0
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import logging
17from collections.abc import Iterator, Sequence
18from typing import IO, Any, Optional, Union
19
20from ..config import StorageClientConfig
21from ..constants import MEMORY_LOAD_LIMIT
22from ..file import ObjectFile, PosixFile
23from ..types import (
24 ExecutionMode,
25 MetadataProvider,
26 ObjectMetadata,
27 PatternList,
28 Range,
29 SignerType,
30 SourceVersionCheckMode,
31 StorageProvider,
32 SymlinkHandling,
33 SyncResult,
34)
35from .composite import CompositeStorageClient
36from .single import SingleStorageClient
37from .types import AbstractStorageClient
38
39logger = logging.getLogger(__name__)
40
41
[docs]
42class StorageClient(AbstractStorageClient):
43 """
44 Unified storage client facade.
45
46 Automatically delegates to:
47 - SingleStorageClient: For single-backend configurations (full read/write)
48 - CompositeStorageClient: For multi-backend configurations (read-only)
49 """
50
51 _delegate: Union[SingleStorageClient, CompositeStorageClient]
52
53 def __init__(self, config: StorageClientConfig):
54 if config.storage_provider_profiles:
55 self._delegate = CompositeStorageClient(config)
56 logger.debug(f"StorageClient '{config.profile}' using CompositeStorageClient (read-only)")
57 else:
58 self._delegate = SingleStorageClient(config)
59 logger.debug(f"StorageClient '{config.profile}' using SingleStorageClient")
60
61 @property
62 def delegate(self) -> Union[SingleStorageClient, CompositeStorageClient]:
63 """
64 Access to underlying delegate storage client.
65
66 :return: SingleStorageClient or CompositeStorageClient.
67 """
68 return self._delegate
69
70 @property
71 def _config(self) -> StorageClientConfig:
72 """
73 :return: The configuration for the underlying storage client.
74 """
75 return self._delegate._config
76
77 @property
78 def _storage_provider(self) -> Optional[StorageProvider]:
79 """
80 :return: The storage provider for the underlying storage client. None for CompositeStorageClient.
81 """
82 return self._delegate._storage_provider
83
84 @_storage_provider.setter
85 def _storage_provider(self, value: StorageProvider) -> None:
86 """Allow mutation of storage provider for testing purposes."""
87 if isinstance(self._delegate, SingleStorageClient):
88 self._delegate._storage_provider = value
89
90 @property
91 def _metadata_provider(self) -> Optional[MetadataProvider]:
92 """
93 :return: The metadata provider for the underlying storage client.
94 """
95 return self._delegate._metadata_provider
96
97 @_metadata_provider.setter
98 def _metadata_provider(self, value: Optional[MetadataProvider]) -> None:
99 """Allow mutation of metadata provider for DSS compatibility."""
100 if isinstance(self._delegate, CompositeStorageClient) and value is None:
101 raise ValueError("CompositeStorageClient requires a metadata_provider for routing decisions.")
102 self._delegate._metadata_provider = value # type: ignore[assignment]
103
104 @property
105 def _metadata_provider_lock(self):
106 """
107 Access to metadata provider lock for DSS compatibility.
108
109 :return: The lock for the metadata provider.
110 """
111 return self._delegate._metadata_provider_lock
112
113 @_metadata_provider_lock.setter
114 def _metadata_provider_lock(self, value):
115 """Allow mutation of metadata provider lock for DSS compatibility."""
116 self._delegate._metadata_provider_lock = value
117
118 @property
119 def _credentials_provider(self):
120 """
121 :return: The credentials provider for the underlying storage client.
122 """
123 return self._delegate._credentials_provider
124
125 @property
126 def _retry_config(self):
127 """
128 :return: The retry configuration for the underlying storage client.
129 """
130 return self._delegate._retry_config
131
132 @property
133 def _cache_manager(self):
134 """
135 :return: The cache manager for the underlying storage client.
136 """
137 return self._delegate._cache_manager
138
139 @property
140 def _replica_manager(self):
141 """
142 :return: The replica manager for the underlying storage client.
143 """
144 return self._delegate._replica_manager
145
146 @_replica_manager.setter
147 def _replica_manager(self, value):
148 """
149 Allow mutation of replica manager for testing purposes.
150
151 :param value: The new replica manager.
152 """
153 if isinstance(self._delegate, SingleStorageClient):
154 self._delegate._replica_manager = value
155
156 @property
157 def profile(self) -> str:
158 """
159 :return: The profile name of the storage client.
160 """
161 return self._delegate.profile
162
163 @property
164 def replicas(self) -> list[AbstractStorageClient]:
165 """
166 :return: List of replica storage clients, sorted by read priority.
167 """
168 return self._delegate.replicas
169
[docs]
170 def is_default_profile(self) -> bool:
171 """
172 :return: ``True`` if the storage client is using the default profile, ``False`` otherwise.
173 """
174 return self._delegate.is_default_profile()
175
176 def _is_rust_client_enabled(self) -> bool:
177 """
178 :return: ``True`` if the storage provider is using the Rust client, ``False`` otherwise.
179 """
180 return self._delegate._is_rust_client_enabled()
181
182 def _is_posix_file_storage_provider(self) -> bool:
183 """
184 :return: ``True`` if the storage client is using a POSIX file storage provider, ``False`` otherwise.
185 """
186 return self._delegate._is_posix_file_storage_provider()
187
[docs]
188 def get_posix_path(self, path: str) -> Optional[str]:
189 """
190 Returns the physical POSIX filesystem path for POSIX storage providers.
191
192 :param path: The path to resolve (may be a symlink or virtual path).
193 :return: Physical POSIX filesystem path if POSIX storage, None otherwise.
194 """
195 return self._delegate.get_posix_path(path)
196
[docs]
197 def read(
198 self,
199 path: str,
200 byte_range: Optional[Range] = None,
201 check_source_version: SourceVersionCheckMode = SourceVersionCheckMode.INHERIT,
202 ) -> bytes:
203 """
204 Read bytes from a file at the specified logical path.
205
206 :param path: The logical path of the object to read.
207 :param byte_range: Optional byte range to read (offset and length).
208 :param check_source_version: Whether to check the source version of cached objects.
209 :return: The content of the object as bytes.
210 :raises FileNotFoundError: If the file at the specified path does not exist.
211 """
212 return self._delegate.read(path, byte_range, check_source_version)
213
[docs]
214 def open(
215 self,
216 path: str,
217 mode: str = "rb",
218 buffering: int = -1,
219 encoding: Optional[str] = None,
220 disable_read_cache: bool = False,
221 memory_load_limit: int = MEMORY_LOAD_LIMIT,
222 atomic: bool = True,
223 check_source_version: SourceVersionCheckMode = SourceVersionCheckMode.INHERIT,
224 attributes: Optional[dict[str, Any]] = None,
225 prefetch_file: Optional[bool] = None,
226 ) -> Union[PosixFile, ObjectFile]:
227 """
228 Open a file for reading or writing.
229
230 :param path: The logical path of the object to open.
231 :param mode: The file mode. Supported modes: "r", "rb", "w", "wb", "a", "ab".
232 :param buffering: The buffering mode. Only applies to PosixFile.
233 :param encoding: The encoding to use for text files.
234 :param disable_read_cache: When set to ``True``, disables caching for file content.
235 This parameter is only applicable to ObjectFile when the mode is "r" or "rb".
236 :param memory_load_limit: Size limit in bytes for loading files into memory. Defaults to 512MB.
237 This parameter is only applicable to ObjectFile when the mode is "r" or "rb". Defaults to 512MB.
238 :param atomic: When set to ``True``, file will be written atomically (rename upon close).
239 This parameter is only applicable to PosixFile in write mode.
240 :param check_source_version: Whether to check the source version of cached objects.
241 :param attributes: Attributes to add to the file.
242 This parameter is only applicable when the mode is "w" or "wb" or "a" or "ab". Defaults to None.
243 :param prefetch_file: Whether to prefetch the file content.
244 This parameter is only applicable to ObjectFile when the mode is "r" or "rb".
245 If None, inherits from cache configuration.
246 :return: A file-like object (PosixFile or ObjectFile) for the specified path.
247 :raises FileNotFoundError: If the file does not exist (read mode).
248 :raises NotImplementedError: If the operation is not supported (e.g., write on CompositeStorageClient).
249 """
250 return self._delegate.open(
251 path,
252 mode,
253 buffering,
254 encoding,
255 disable_read_cache,
256 memory_load_limit,
257 atomic,
258 check_source_version,
259 attributes,
260 prefetch_file,
261 )
262
[docs]
263 def download_file(self, remote_path: str, local_path: Union[str, IO]) -> None:
264 """
265 Download a remote file to a local path or file-like object.
266
267 :param remote_path: The logical path of the remote file to download.
268 :param local_path: The local file path or file-like object to write to.
269 :raises FileNotFoundError: If the remote file does not exist.
270 """
271 return self._delegate.download_file(remote_path, local_path)
272
[docs]
273 def download_files(
274 self,
275 remote_paths: list[str],
276 local_paths: list[str],
277 metadata: Optional[Sequence[Optional[ObjectMetadata]]] = None,
278 max_workers: int = 16,
279 ) -> None:
280 """
281 Download multiple remote files to local paths.
282
283 :param remote_paths: List of logical paths of remote files to download.
284 :param local_paths: List of local file paths to save the downloaded files to.
285 :param metadata: Optional per-file metadata used to decide between regular and multipart download.
286 :param max_workers: Maximum number of concurrent download workers (default: 16).
287 :raises ValueError: If remote_paths and local_paths have different lengths.
288 :raises FileNotFoundError: If any remote file does not exist.
289 """
290 return self._delegate.download_files(remote_paths, local_paths, metadata, max_workers)
291
[docs]
292 def glob(
293 self,
294 pattern: str,
295 include_url_prefix: bool = False,
296 attribute_filter_expression: Optional[str] = None,
297 ) -> list[str]:
298 """
299 Matches and retrieves a list of object keys in the storage provider that match the specified pattern.
300
301 :param pattern: The pattern to match object keys against, supporting wildcards (e.g., ``*.txt``).
302 :param include_url_prefix: Whether to include the URL prefix ``msc://profile`` in the result.
303 :param attribute_filter_expression: The attribute filter expression to apply to the result.
304 :return: A list of object paths that match the specified pattern.
305 """
306 return self._delegate.glob(pattern, include_url_prefix, attribute_filter_expression)
307
[docs]
308 def list_recursive(
309 self,
310 path: str = "",
311 start_after: Optional[str] = None,
312 end_at: Optional[str] = None,
313 max_workers: int = 32,
314 look_ahead: int = 2,
315 include_url_prefix: bool = False,
316 follow_symlinks: Optional[bool] = None,
317 patterns: Optional[PatternList] = None,
318 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
319 ) -> Iterator[ObjectMetadata]:
320 """
321 List files recursively in the storage provider under the specified path.
322
323 :param path: The directory or file path to list objects under. This should be a
324 complete filesystem path (e.g., "my-bucket/documents/" or "data/2024/").
325 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist.
326 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist.
327 :param max_workers: Maximum concurrent workers for provider-level recursive listing.
328 :param look_ahead: Prefixes to buffer per worker for provider-level recursive listing.
329 :param include_url_prefix: Whether to include the URL prefix ``msc://profile`` in the result.
330 :param follow_symlinks: **Deprecated.** Use ``symlink_handling`` instead.
331 :param patterns: PatternList for include/exclude filtering. If None, all files are included.
332 :param symlink_handling: How to handle symbolic links during listing. Only applicable for POSIX file storage providers.
333 :return: An iterator over ObjectMetadata for matching files.
334 """
335 return self._delegate.list_recursive(
336 path=path,
337 start_after=start_after,
338 end_at=end_at,
339 max_workers=max_workers,
340 look_ahead=look_ahead,
341 include_url_prefix=include_url_prefix,
342 follow_symlinks=follow_symlinks,
343 patterns=patterns,
344 symlink_handling=symlink_handling,
345 )
346
[docs]
347 def is_file(self, path: str) -> bool:
348 """
349 Checks whether the specified path points to a file (rather than a folder or directory).
350
351 :param path: The logical path to check.
352 :return: ``True`` if the key points to a file, ``False`` otherwise.
353 """
354 return self._delegate.is_file(path)
355
[docs]
356 def is_empty(self, path: str) -> bool:
357 """
358 Check whether the specified path is empty. A path is considered empty if there are no
359 objects whose keys start with the given path as a prefix.
360
361 :param path: The logical path to check (typically a directory or folder prefix).
362 :return: ``True`` if no objects exist under the specified path prefix, ``False`` otherwise.
363 """
364 return self._delegate.is_empty(path)
365
[docs]
366 def info(self, path: str, strict: bool = True) -> ObjectMetadata:
367 """
368 Get metadata for a file at the specified path.
369
370 :param path: The logical path of the object.
371 :param strict: When ``True``, only return committed metadata. When ``False``, include pending changes.
372 :return: ObjectMetadata containing file information (size, last modified, etc.).
373 :raises FileNotFoundError: If the file at the specified path does not exist.
374 """
375 return self._delegate.info(path, strict)
376
[docs]
377 def write(
378 self,
379 path: str,
380 body: bytes,
381 attributes: Optional[dict[str, Any]] = None,
382 ) -> None:
383 """
384 Write bytes to a file at the specified path.
385
386 :param path: The logical path where the object will be written.
387 :param body: The content to write as bytes.
388 :param attributes: Optional attributes to add to the file.
389 :raises NotImplementedError: If write operations are not supported (e.g., CompositeStorageClient).
390 """
391 return self._delegate.write(path, body, attributes)
392
[docs]
393 def delete(self, path: str, recursive: bool = False) -> None:
394 """
395 Delete a file or directory at the specified path.
396
397 :param path: The logical path of the object to delete.
398 :param recursive: When True, delete directory and all its contents recursively.
399 :raises FileNotFoundError: If the file or directory does not exist.
400 :raises NotImplementedError: If delete operations are not supported (e.g., CompositeStorageClient).
401 """
402 return self._delegate.delete(path, recursive)
403
[docs]
404 def delete_many(self, paths: list[str]) -> None:
405 """
406 Delete multiple files at the specified paths. Only files are supported; directories are not deleted.
407
408 :param paths: List of logical paths of the files to delete.
409 :raises NotImplementedError: If delete operations are not supported (e.g., CompositeStorageClient).
410 """
411 return self._delegate.delete_many(paths)
412
[docs]
413 def copy(self, src_path: str, dest_path: str) -> None:
414 """
415 Copy a file from source path to destination path.
416
417 :param src_path: The logical path of the source object.
418 :param dest_path: The logical path where the object will be copied to.
419 :raises FileNotFoundError: If the source file does not exist.
420 :raises NotImplementedError: If copy operations are not supported (e.g., CompositeStorageClient).
421 """
422 return self._delegate.copy(src_path, dest_path)
423
[docs]
424 def make_symlink(self, path: str, target: str) -> None:
425 """
426 Creates a symbolic link at ``path`` pointing to ``target``.
427
428 On POSIX backends this creates a native OS symlink with a relative target path.
429 On object-store backends this creates a zero-byte marker object with the target
430 stored in user metadata.
431
432 :param path: The logical path where the symlink will be created.
433 :param target: The logical key that the symlink points to.
434 :raises NotImplementedError: If symlink creation is not supported.
435 """
436 return self._delegate.make_symlink(path, target)
437
[docs]
438 def upload_file(
439 self,
440 remote_path: str,
441 local_path: Union[str, IO],
442 attributes: Optional[dict[str, Any]] = None,
443 ) -> None:
444 """
445 Upload a local file to remote storage.
446
447 :param remote_path: The logical path where the file will be uploaded.
448 :param local_path: The local file path or file-like object to upload.
449 :param attributes: Optional attributes to add to the file.
450 :raises FileNotFoundError: If the local file does not exist.
451 :raises NotImplementedError: If upload operations are not supported (e.g., CompositeStorageClient).
452 """
453 return self._delegate.upload_file(remote_path, local_path, attributes)
454
[docs]
455 def upload_files(
456 self,
457 remote_paths: list[str],
458 local_paths: list[str],
459 attributes: Optional[Sequence[Optional[dict[str, Any]]]] = None,
460 max_workers: int = 16,
461 ) -> None:
462 """
463 Upload multiple local files to remote storage.
464
465 :param remote_paths: List of logical paths where the files will be uploaded.
466 :param local_paths: List of local file paths to upload.
467 :param attributes: Optional list of per-file attributes to add. When provided, must have the same length
468 as remote_paths/local_paths. Each element may be ``None`` for files that need no attributes.
469 :param max_workers: Maximum number of concurrent upload workers (default: 16).
470 :raises ValueError: If remote_paths and local_paths have different lengths.
471 :raises ValueError: If attributes is provided and has a different length than remote_paths.
472 :raises NotImplementedError: If upload operations are not supported (e.g., CompositeStorageClient).
473 """
474 return self._delegate.upload_files(remote_paths, local_paths, attributes, max_workers)
475
483
[docs]
484 def sync_from(
485 self,
486 source_client: AbstractStorageClient,
487 source_path: str = "",
488 target_path: str = "",
489 delete_unmatched_files: bool = False,
490 description: str = "Syncing",
491 num_worker_processes: Optional[int] = None,
492 execution_mode: ExecutionMode = ExecutionMode.LOCAL,
493 patterns: Optional[PatternList] = None,
494 preserve_source_attributes: bool = False,
495 follow_symlinks: Optional[bool] = None,
496 source_files: Optional[list[str]] = None,
497 ignore_hidden: bool = True,
498 commit_metadata: bool = True,
499 dryrun: bool = False,
500 dryrun_output_path: Optional[str] = None,
501 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
502 ) -> SyncResult:
503 """
504 Syncs files from the source storage client to "path/".
505
506 :param source_client: The source storage client.
507 :param source_path: The logical path to sync from.
508 :param target_path: The logical path to sync to.
509 :param delete_unmatched_files: Whether to delete files at the target that are not present at the source.
510 :param description: Description of sync process for logging purposes.
511 :param num_worker_processes: The number of worker processes to use.
512 :param execution_mode: The execution mode to use. Currently supports "local" and "ray".
513 :param patterns: PatternList for include/exclude filtering. If None, all files are included.
514 Cannot be used together with source_files.
515 :param preserve_source_attributes: Whether to preserve source file metadata attributes during synchronization.
516 When ``False`` (default), only file content is copied. When ``True``, custom metadata attributes are also preserved.
517
518 .. warning::
519 **Performance Impact**: When enabled without a ``metadata_provider`` configured, this will make a HEAD
520 request for each object to retrieve attributes, which can significantly impact performance on large-scale
521 sync operations. For production use at scale, configure a ``metadata_provider`` in your storage profile.
522
523 :param follow_symlinks: **Deprecated.** Use ``symlink_handling`` instead. If the source StorageClient is PosixFile,
524 whether to follow symbolic links. ``True`` maps to :py:attr:`SymlinkHandling.FOLLOW`; ``False`` maps to
525 :py:attr:`SymlinkHandling.SKIP`. Cannot be combined with a non-default ``symlink_handling``.
526 :param source_files: Optional list of file paths (relative to source_path) to sync. When provided, only these
527 specific files will be synced, skipping enumeration of the source path. Cannot be used together with patterns.
528 :param ignore_hidden: Whether to ignore hidden files and directories. Default is ``True``.
529 :param commit_metadata: When ``True`` (default), calls :py:meth:`StorageClient.commit_metadata` after sync completes.
530 Set to ``False`` to skip the commit, allowing batching of multiple sync operations before committing manually.
531 :param dryrun: If ``True``, only enumerate and compare objects without performing any copy/delete operations.
532 The returned :py:class:`SyncResult` will include a :py:class:`DryrunResult` with paths to JSONL files.
533 :param dryrun_output_path: Directory to write dryrun JSONL files into. If ``None`` (default), a temporary
534 directory is created automatically. Ignored when ``dryrun`` is ``False``.
535 :param symlink_handling: How to handle symbolic links during sync.
536 :py:attr:`SymlinkHandling.FOLLOW` (default) dereferences symlinks and copies the target's bytes.
537 :py:attr:`SymlinkHandling.SKIP` excludes symlinks from the sync.
538 :py:attr:`SymlinkHandling.PRESERVE` recreates symlinks on the target via :py:meth:`make_symlink`
539 instead of copying bytes (required for round-trip preservation of symlinks).
540 :raises ValueError: If both source_files and patterns are provided, or if ``follow_symlinks`` is combined
541 with a non-default ``symlink_handling``.
542 :raises NotImplementedError: If sync operations are not supported (e.g., CompositeStorageClient as target).
543 """
544 return self._delegate.sync_from(
545 source_client,
546 source_path,
547 target_path,
548 delete_unmatched_files,
549 description,
550 num_worker_processes,
551 execution_mode,
552 patterns,
553 preserve_source_attributes,
554 follow_symlinks,
555 source_files,
556 ignore_hidden,
557 commit_metadata,
558 dryrun,
559 dryrun_output_path,
560 symlink_handling,
561 )
562
[docs]
563 def sync_replicas(
564 self,
565 source_path: str,
566 replica_indices: Optional[list[int]] = None,
567 delete_unmatched_files: bool = False,
568 description: str = "Syncing replica",
569 num_worker_processes: Optional[int] = None,
570 execution_mode: ExecutionMode = ExecutionMode.LOCAL,
571 patterns: Optional[PatternList] = None,
572 ignore_hidden: bool = True,
573 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
574 ) -> None:
575 """
576 Sync files from this client to its replica storage clients.
577
578 :param source_path: The logical path to sync from.
579 :param replica_indices: Specific replica indices to sync to (0-indexed). If None, syncs to all replicas.
580 :param delete_unmatched_files: When set to ``True``, delete files in replicas that don't exist in source.
581 :param description: Description of sync process for logging purposes.
582 :param num_worker_processes: Number of worker processes for parallel sync.
583 :param execution_mode: Execution mode (LOCAL or REMOTE).
584 :param patterns: PatternList for include/exclude filtering. If None, all files are included.
585 :param ignore_hidden: When set to ``True``, ignore hidden files (starting with '.'). Defaults to ``True``.
586 :param symlink_handling: How to handle symbolic links during sync.
587 :py:attr:`SymlinkHandling.FOLLOW` (default) dereferences symlinks and copies the target's bytes.
588 :py:attr:`SymlinkHandling.SKIP` excludes symlinks from the sync.
589 :py:attr:`SymlinkHandling.PRESERVE` recreates symlinks on each replica via
590 :py:meth:`make_symlink` instead of copying bytes.
591 """
592 return self._delegate.sync_replicas(
593 source_path,
594 replica_indices,
595 delete_unmatched_files,
596 description,
597 num_worker_processes,
598 execution_mode,
599 patterns,
600 ignore_hidden,
601 symlink_handling,
602 )
603
[docs]
604 def list(
605 self,
606 prefix: str = "",
607 path: str = "",
608 start_after: Optional[str] = None,
609 end_at: Optional[str] = None,
610 include_directories: bool = False,
611 include_url_prefix: bool = False,
612 attribute_filter_expression: Optional[str] = None,
613 show_attributes: bool = False,
614 follow_symlinks: Optional[bool] = None,
615 patterns: Optional[PatternList] = None,
616 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
617 ) -> Iterator[ObjectMetadata]:
618 """
619 List objects in the storage provider under the specified path.
620
621 **IMPORTANT**: Use the ``path`` parameter for new code. The ``prefix`` parameter is
622 deprecated and will be removed in a future version.
623
624 :param prefix: [DEPRECATED] Use ``path`` instead. The prefix to list objects under.
625 :param path: The directory or file path to list objects under. This should be a
626 complete filesystem path (e.g., "my-bucket/documents/" or "data/2024/").
627 Cannot be used together with ``prefix``.
628 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist.
629 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist.
630 :param include_directories: Whether to include directories in the result. When ``True``, directories are returned alongside objects.
631 :param include_url_prefix: Whether to include the URL prefix ``msc://profile`` in the result.
632 :param attribute_filter_expression: The attribute filter expression to apply to the result.
633 :param show_attributes: Whether to return attributes in the result. WARNING: Depending on implementation, there may be a performance impact if this is set to ``True``.
634 :param follow_symlinks: **Deprecated.** Use ``symlink_handling`` instead.
635 :param patterns: PatternList for include/exclude filtering. If None, all files are included.
636 :param symlink_handling: How to handle symbolic links during listing. Only applicable for POSIX file storage providers.
637 :return: An iterator over ObjectMetadata for matching objects.
638 :raises ValueError: If both ``path`` and ``prefix`` parameters are provided (both non-empty).
639 """
640 return self._delegate.list(
641 prefix,
642 path,
643 start_after,
644 end_at,
645 include_directories,
646 include_url_prefix,
647 attribute_filter_expression,
648 show_attributes,
649 follow_symlinks=follow_symlinks,
650 patterns=patterns,
651 symlink_handling=symlink_handling,
652 )
653
[docs]
654 def generate_presigned_url(
655 self,
656 path: str,
657 *,
658 method: str = "GET",
659 signer_type: Optional[SignerType] = None,
660 signer_options: Optional[dict[str, Any]] = None,
661 ) -> str:
662 """
663 Generate a pre-signed URL granting temporary access to the object at *path*.
664
665 :param path: The logical path of the object.
666 :param method: The HTTP method the URL should authorise (e.g. ``"GET"``, ``"PUT"``).
667 :param signer_type: The signing backend to use. ``None`` means the provider's native signer.
668 :param signer_options: Backend-specific options forwarded to the signer.
669 :return: A pre-signed URL string.
670 :raises NotImplementedError: If the underlying storage provider does not support presigned URLs.
671 """
672 return self._delegate.generate_presigned_url(
673 path, method=method, signer_type=signer_type, signer_options=signer_options
674 )
675
676 def __getstate__(self) -> dict[str, Any]:
677 """Support for pickling (forward to delegate)."""
678 return self._delegate.__getstate__()
679
680 def __setstate__(self, state: dict[str, Any]) -> None:
681 """Support for unpickling - reconstruct the delegate."""
682 config = state["_config"]
683
684 if config.storage_provider_profiles:
685 self._delegate = CompositeStorageClient.__new__(CompositeStorageClient)
686 self._delegate.__setstate__(state)
687 else:
688 self._delegate = SingleStorageClient.__new__(SingleStorageClient)
689 self._delegate.__setstate__(state)