1# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2# SPDX-License-Identifier: Apache-2.0
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import logging
17from collections.abc import Iterator, Sequence
18from typing import IO, Any
19
20from ..config import StorageClientConfig
21from ..constants import MEMORY_LOAD_LIMIT
22from ..file import ObjectFile, PosixFile
23from ..types import (
24 BatchTransferCallback,
25 ExecutionMode,
26 MetadataProvider,
27 ObjectMetadata,
28 PatternList,
29 Range,
30 SignerType,
31 SourceVersionCheckMode,
32 StorageProvider,
33 SuffixRange,
34 SymlinkHandling,
35 SyncResult,
36)
37from ..utils import _invoke_batch_transfer
38from .composite import CompositeStorageClient
39from .single import SingleStorageClient
40from .types import AbstractStorageClient
41
42logger = logging.getLogger(__name__)
43
44
[docs]
45class StorageClient(AbstractStorageClient):
46 """
47 Unified storage client facade.
48
49 Automatically delegates to:
50 - SingleStorageClient: For single-backend configurations (full read/write)
51 - CompositeStorageClient: For multi-backend configurations (read-only)
52 """
53
54 _delegate: SingleStorageClient | CompositeStorageClient
55
56 def __init__(self, config: StorageClientConfig):
57 if config.storage_provider_profiles:
58 self._delegate = CompositeStorageClient(config)
59 logger.debug(f"StorageClient '{config.profile}' using CompositeStorageClient (read-only)")
60 else:
61 self._delegate = SingleStorageClient(config)
62 logger.debug(f"StorageClient '{config.profile}' using SingleStorageClient")
63
64 @property
65 def delegate(self) -> SingleStorageClient | CompositeStorageClient:
66 """
67 Access to underlying delegate storage client.
68
69 :return: SingleStorageClient or CompositeStorageClient.
70 """
71 return self._delegate
72
73 @property
74 def _config(self) -> StorageClientConfig:
75 """
76 :return: The configuration for the underlying storage client.
77 """
78 return self._delegate._config
79
80 @property
81 def _storage_provider(self) -> StorageProvider | None:
82 """
83 :return: The storage provider for the underlying storage client. None for CompositeStorageClient.
84 """
85 return self._delegate._storage_provider
86
87 @_storage_provider.setter
88 def _storage_provider(self, value: StorageProvider) -> None:
89 """Allow mutation of storage provider for testing purposes."""
90 if isinstance(self._delegate, SingleStorageClient):
91 self._delegate._storage_provider = value
92
93 @property
94 def _metadata_provider(self) -> MetadataProvider | None:
95 """
96 :return: The metadata provider for the underlying storage client.
97 """
98 return self._delegate._metadata_provider
99
100 @_metadata_provider.setter
101 def _metadata_provider(self, value: MetadataProvider | None) -> None:
102 """Allow mutation of metadata provider for DSS compatibility."""
103 if isinstance(self._delegate, CompositeStorageClient) and value is None:
104 raise ValueError("CompositeStorageClient requires a metadata_provider for routing decisions.")
105 self._delegate._metadata_provider = value # type: ignore[assignment]
106
107 @property
108 def _metadata_provider_lock(self):
109 """
110 Access to metadata provider lock for DSS compatibility.
111
112 :return: The lock for the metadata provider.
113 """
114 return self._delegate._metadata_provider_lock
115
116 @_metadata_provider_lock.setter
117 def _metadata_provider_lock(self, value):
118 """Allow mutation of metadata provider lock for DSS compatibility."""
119 self._delegate._metadata_provider_lock = value
120
121 @property
122 def _credentials_provider(self):
123 """
124 :return: The credentials provider for the underlying storage client.
125 """
126 return self._delegate._credentials_provider
127
128 @property
129 def _retry_config(self):
130 """
131 :return: The retry configuration for the underlying storage client.
132 """
133 return self._delegate._retry_config
134
135 @property
136 def _cache_manager(self):
137 """
138 :return: The cache manager for the underlying storage client.
139 """
140 return self._delegate._cache_manager
141
142 @property
143 def _replica_manager(self):
144 """
145 :return: The replica manager for the underlying storage client.
146 """
147 return self._delegate._replica_manager
148
149 @_replica_manager.setter
150 def _replica_manager(self, value):
151 """
152 Allow mutation of replica manager for testing purposes.
153
154 :param value: The new replica manager.
155 """
156 if isinstance(self._delegate, SingleStorageClient):
157 self._delegate._replica_manager = value
158
159 @property
160 def profile(self) -> str:
161 """
162 :return: The profile name of the storage client.
163 """
164 return self._delegate.profile
165
166 @property
167 def replicas(self) -> list[AbstractStorageClient]:
168 """
169 :return: List of replica storage clients, sorted by read priority.
170 """
171 return self._delegate.replicas
172
[docs]
173 def is_default_profile(self) -> bool:
174 """
175 :return: ``True`` if the storage client is using the default profile, ``False`` otherwise.
176 """
177 return self._delegate.is_default_profile()
178
179 def _is_rust_client_enabled(self) -> bool:
180 """
181 :return: ``True`` if the storage provider is using the Rust client, ``False`` otherwise.
182 """
183 return self._delegate._is_rust_client_enabled()
184
185 def _is_posix_file_storage_provider(self) -> bool:
186 """
187 :return: ``True`` if the storage client is using a POSIX file storage provider, ``False`` otherwise.
188 """
189 return self._delegate._is_posix_file_storage_provider()
190
[docs]
191 def get_posix_path(self, path: str) -> str | None:
192 """
193 Returns the physical POSIX filesystem path for POSIX storage providers.
194
195 :param path: The path to resolve (may be a symlink or virtual path).
196 :return: Physical POSIX filesystem path if POSIX storage, None otherwise.
197 """
198 return self._delegate.get_posix_path(path)
199
[docs]
200 def read(
201 self,
202 path: str,
203 byte_range: Range | SuffixRange | None = None,
204 check_source_version: SourceVersionCheckMode = SourceVersionCheckMode.INHERIT,
205 ) -> bytes:
206 """
207 Read bytes from a file at the specified logical path.
208
209 :param path: The logical path of the object to read.
210 :param byte_range: Optional byte range to read (offset and length), or a suffix range to read the last bytes.
211 :param check_source_version: Whether to check the source version of cached objects.
212 :return: The content of the object as bytes.
213 :raises FileNotFoundError: If the file at the specified path does not exist.
214 """
215 return self._delegate.read(path, byte_range, check_source_version)
216
[docs]
217 def open(
218 self,
219 path: str,
220 mode: str = "rb",
221 buffering: int = -1,
222 encoding: str | None = None,
223 disable_read_cache: bool = False,
224 memory_load_limit: int = MEMORY_LOAD_LIMIT,
225 atomic: bool = True,
226 check_source_version: SourceVersionCheckMode = SourceVersionCheckMode.INHERIT,
227 attributes: dict[str, Any] | None = None,
228 prefetch_file: bool | None = None,
229 ) -> PosixFile | ObjectFile:
230 """
231 Open a file for reading or writing.
232
233 :param path: The logical path of the object to open.
234 :param mode: The file mode. Supported modes: "r", "rb", "w", "wb", "a", "ab".
235 :param buffering: The buffering mode. Only applies to PosixFile.
236 :param encoding: The encoding to use for text files.
237 :param disable_read_cache: When set to ``True``, disables caching for file content.
238 This parameter is only applicable to ObjectFile when the mode is "r" or "rb".
239 :param memory_load_limit: Size limit in bytes for loading files into memory. Defaults to 512MB.
240 This parameter is only applicable to ObjectFile when the mode is "r" or "rb". Defaults to 512MB.
241 :param atomic: When set to ``True``, file will be written atomically (rename upon close).
242 This parameter is only applicable to PosixFile in write mode.
243 :param check_source_version: Whether to check the source version of cached objects.
244 :param attributes: Attributes to add to the file.
245 This parameter is only applicable when the mode is "w" or "wb" or "a" or "ab". Defaults to None.
246 :param prefetch_file: Whether to prefetch the file content.
247 This parameter is only applicable to ObjectFile when the mode is "r" or "rb".
248 If None, inherits from cache configuration.
249 :return: A file-like object (PosixFile or ObjectFile) for the specified path.
250 :raises FileNotFoundError: If the file does not exist (read mode).
251 :raises NotImplementedError: If the operation is not supported (e.g., write on CompositeStorageClient).
252 """
253 return self._delegate.open(
254 path,
255 mode,
256 buffering,
257 encoding,
258 disable_read_cache,
259 memory_load_limit,
260 atomic,
261 check_source_version,
262 attributes,
263 prefetch_file,
264 )
265
[docs]
266 def download_file(self, remote_path: str, local_path: str | IO, metadata: ObjectMetadata | None = None) -> None:
267 """
268 Download a remote file to a local path or file-like object.
269
270 :param remote_path: The logical path of the remote file to download.
271 :param local_path: The local file path or file-like object to write to.
272 :param metadata: Optional metadata for this object, as returned by :py:meth:`info`, to avoid a provider lookup.
273 :raises FileNotFoundError: If the remote file does not exist.
274 :raises PermissionError: If the provider reports denied read access.
275 :raises multistorageclient.types.RetryableError: If a retryable transfer failure remains after configured retries.
276 """
277 return self._delegate.download_file(remote_path, local_path, metadata)
278
[docs]
279 def download_files(
280 self,
281 remote_paths: Sequence[str],
282 local_paths: Sequence[str],
283 metadata: Sequence[ObjectMetadata | None] | None = None,
284 max_workers: int = 16,
285 transfer_callback: BatchTransferCallback | None = None,
286 ) -> None:
287 """
288 Download multiple remote files to local paths.
289
290 :param remote_paths: List of logical paths of remote files to download.
291 :param local_paths: List of local file paths to save the downloaded files to.
292 :param metadata: Optional per-file metadata used to decide between regular and multipart download.
293 :param max_workers: Maximum number of concurrent download workers (default: 16).
294 :param transfer_callback: Optional observer called with source path, destination path, and "started",
295 "completed", or "failed" for each executing attempt. Remote paths may be resolved physical paths.
296 Calls may be concurrent; callback exceptions are logged.
297 :raises ValueError: If remote_paths and local_paths have different lengths.
298 :raises FileNotFoundError: If any remote file does not exist.
299 """
300 return _invoke_batch_transfer(
301 self._delegate.download_files,
302 remote_paths,
303 local_paths,
304 metadata,
305 max_workers,
306 transfer_callback=transfer_callback,
307 )
308
[docs]
309 def glob(
310 self,
311 pattern: str,
312 include_url_prefix: bool = False,
313 attribute_filter_expression: str | None = None,
314 ) -> list[str]:
315 """
316 Matches and retrieves a list of object keys in the storage provider that match the specified pattern.
317
318 :param pattern: The pattern to match object keys against, supporting wildcards (e.g., ``*.txt``).
319 :param include_url_prefix: Whether to include the URL prefix ``msc://profile`` in the result.
320 :param attribute_filter_expression: The attribute filter expression to apply to the result.
321 :return: A list of object paths that match the specified pattern.
322 """
323 return self._delegate.glob(pattern, include_url_prefix, attribute_filter_expression)
324
[docs]
325 def list_recursive(
326 self,
327 path: str = "",
328 start_after: str | None = None,
329 end_at: str | None = None,
330 max_workers: int = 32,
331 look_ahead: int = 2,
332 include_url_prefix: bool = False,
333 patterns: PatternList | None = None,
334 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
335 ) -> Iterator[ObjectMetadata]:
336 """
337 List files recursively in the storage provider under the specified path.
338
339 :param path: The directory or file path to list objects under. This should be a
340 complete filesystem path (e.g., "my-bucket/documents/" or "data/2024/").
341 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist.
342 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist.
343 :param max_workers: Maximum concurrent workers for provider-level recursive listing.
344 :param look_ahead: Prefixes to buffer per worker for provider-level recursive listing.
345 :param include_url_prefix: Whether to include the URL prefix ``msc://profile`` in the result.
346 :param patterns: PatternList for include/exclude filtering. If None, all files are included.
347 :param symlink_handling: How to handle symbolic links during listing. Only applicable for POSIX file storage providers.
348 :return: An iterator over ObjectMetadata for matching files.
349 """
350 return self._delegate.list_recursive(
351 path=path,
352 start_after=start_after,
353 end_at=end_at,
354 max_workers=max_workers,
355 look_ahead=look_ahead,
356 include_url_prefix=include_url_prefix,
357 patterns=patterns,
358 symlink_handling=symlink_handling,
359 )
360
[docs]
361 def is_file(self, path: str) -> bool:
362 """
363 Checks whether the specified path points to a file (rather than a folder or directory).
364
365 :param path: The logical path to check.
366 :return: ``True`` if the key points to a file, ``False`` otherwise.
367 """
368 return self._delegate.is_file(path)
369
[docs]
370 def is_empty(self, path: str) -> bool:
371 """
372 Check whether the specified path is empty. A path is considered empty if there are no
373 objects whose keys start with the given path as a prefix.
374
375 :param path: The logical path to check (typically a directory or folder prefix).
376 :return: ``True`` if no objects exist under the specified path prefix, ``False`` otherwise.
377 """
378 return self._delegate.is_empty(path)
379
[docs]
380 def info(self, path: str, strict: bool = True) -> ObjectMetadata:
381 """
382 Get metadata for a file at the specified path.
383
384 :param path: The logical path of the object.
385 :param strict: When ``True``, only return committed metadata. When ``False``, include pending changes.
386 :return: ObjectMetadata containing file information (size, last modified, etc.).
387 :raises FileNotFoundError: If the file at the specified path does not exist.
388 """
389 return self._delegate.info(path, strict)
390
[docs]
391 def write(
392 self,
393 path: str,
394 body: bytes,
395 attributes: dict[str, Any] | None = None,
396 ) -> None:
397 """
398 Write bytes to a file at the specified path.
399
400 :param path: The logical path where the object will be written.
401 :param body: The content to write as bytes.
402 :param attributes: Optional attributes to add to the file.
403 :raises NotImplementedError: If write operations are not supported (e.g., CompositeStorageClient).
404 """
405 return self._delegate.write(path, body, attributes)
406
[docs]
407 def delete(self, path: str, recursive: bool = False) -> None:
408 """
409 Delete a file or directory at the specified path.
410
411 :param path: The logical path of the object to delete.
412 :param recursive: When True, delete directory and all its contents recursively.
413 :raises FileNotFoundError: If the file or directory does not exist.
414 :raises NotImplementedError: If delete operations are not supported (e.g., CompositeStorageClient).
415 """
416 return self._delegate.delete(path, recursive)
417
[docs]
418 def delete_many(self, paths: Sequence[str]) -> None:
419 """
420 Delete multiple files at the specified paths. Only files are supported; directories are not deleted.
421 Paths that do not exist are treated as successful no-ops.
422
423 :param paths: List of logical paths of the files to delete.
424 :raises NotImplementedError: If delete operations are not supported (e.g., CompositeStorageClient).
425 """
426 return self._delegate.delete_many(paths)
427
[docs]
428 def copy(self, src_path: str, dest_path: str) -> None:
429 """
430 Copy a file from source path to destination path.
431
432 :param src_path: The logical path of the source object.
433 :param dest_path: The logical path where the object will be copied to.
434 :raises FileNotFoundError: If the source file does not exist.
435 :raises NotImplementedError: If copy operations are not supported (e.g., CompositeStorageClient).
436 """
437 return self._delegate.copy(src_path, dest_path)
438
[docs]
439 def make_symlink(self, path: str, target: str) -> None:
440 """
441 Creates a symbolic link at ``path`` pointing to ``target``.
442
443 On POSIX backends this creates a native OS symlink with a relative target path.
444 On object-store backends this creates a zero-byte marker object with the target
445 stored in user metadata.
446
447 :param path: The logical path where the symlink will be created.
448 :param target: The logical key that the symlink points to.
449 :raises NotImplementedError: If symlink creation is not supported.
450 """
451 return self._delegate.make_symlink(path, target)
452
[docs]
453 def upload_file(
454 self,
455 remote_path: str,
456 local_path: str | IO,
457 attributes: dict[str, Any] | None = None,
458 ) -> None:
459 """
460 Upload a local file to remote storage.
461
462 :param remote_path: The logical path where the file will be uploaded.
463 :param local_path: The local file path or file-like object to upload.
464 :param attributes: Optional attributes to add to the file.
465 :raises FileNotFoundError: If the local file does not exist.
466 :raises NotImplementedError: If upload operations are not supported (e.g., CompositeStorageClient).
467 """
468 return self._delegate.upload_file(remote_path, local_path, attributes)
469
[docs]
470 def upload_files(
471 self,
472 remote_paths: Sequence[str],
473 local_paths: Sequence[str],
474 attributes: Sequence[dict[str, Any] | None] | None = None,
475 max_workers: int = 16,
476 transfer_callback: BatchTransferCallback | None = None,
477 ) -> None:
478 """
479 Upload multiple local files to remote storage.
480
481 :param remote_paths: List of logical paths where the files will be uploaded.
482 :param local_paths: List of local file paths to upload.
483 :param attributes: Optional list of per-file attributes to add. When provided, must have the same length
484 as remote_paths/local_paths. Each element may be ``None`` for files that need no attributes.
485 :param max_workers: Maximum number of concurrent upload workers (default: 16).
486 :param transfer_callback: Optional observer called with source path, destination path, and "started",
487 "completed", or "failed" for each executing attempt. Remote paths may be resolved physical paths.
488 Calls may be concurrent; callback exceptions are logged.
489 :raises ValueError: If remote_paths and local_paths have different lengths.
490 :raises ValueError: If attributes is provided and has a different length than remote_paths.
491 :raises NotImplementedError: If upload operations are not supported (e.g., CompositeStorageClient).
492 """
493 return _invoke_batch_transfer(
494 self._delegate.upload_files,
495 remote_paths,
496 local_paths,
497 attributes,
498 max_workers,
499 transfer_callback=transfer_callback,
500 )
501
509
[docs]
510 def sync_from(
511 self,
512 source_client: AbstractStorageClient,
513 source_path: str = "",
514 target_path: str = "",
515 delete_unmatched_files: bool = False,
516 description: str = "Syncing",
517 num_worker_processes: int | None = None,
518 execution_mode: ExecutionMode = ExecutionMode.LOCAL,
519 patterns: PatternList | None = None,
520 preserve_source_attributes: bool = False,
521 source_files: list[str] | None = None,
522 ignore_hidden: bool = True,
523 commit_metadata: bool = True,
524 dryrun: bool = False,
525 dryrun_output_path: str | None = None,
526 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
527 ) -> SyncResult:
528 """
529 Syncs files from the source storage client to "path/".
530
531 :param source_client: The source storage client.
532 :param source_path: The logical path to sync from.
533 :param target_path: The logical path to sync to.
534 :param delete_unmatched_files: Whether to delete files at the target that are not present at the source.
535 :param description: Description of sync process for logging purposes.
536 :param num_worker_processes: The number of worker processes to use.
537 :param execution_mode: The execution mode to use. Currently supports "local" and "ray".
538 :param patterns: PatternList for include/exclude filtering. If None, all files are included.
539 Cannot be used together with source_files.
540 :param preserve_source_attributes: Whether to preserve source file metadata attributes during synchronization.
541 When ``False`` (default), only file content is copied. When ``True``, custom metadata attributes are also preserved.
542
543 .. warning::
544 **Performance Impact**: When enabled without a ``metadata_provider`` configured, this will make a HEAD
545 request for each object to retrieve attributes, which can significantly impact performance on large-scale
546 sync operations. For production use at scale, configure a ``metadata_provider`` in your storage profile.
547
548 :param source_files: Optional list of file paths (relative to source_path) to sync. When provided, only these
549 specific files will be synced, skipping enumeration of the source path. Cannot be used together with patterns.
550 :param ignore_hidden: Whether to ignore hidden files and directories. Default is ``True``.
551 :param commit_metadata: When ``True`` (default), calls :py:meth:`StorageClient.commit_metadata` after sync completes.
552 Set to ``False`` to skip the commit, allowing batching of multiple sync operations before committing manually.
553 :param dryrun: If ``True``, only enumerate and compare objects without performing any copy/delete operations.
554 The returned :py:class:`SyncResult` will include a :py:class:`DryrunResult` with paths to JSONL files.
555 :param dryrun_output_path: Directory to write dryrun JSONL files into. If ``None`` (default), a temporary
556 directory is created automatically. Ignored when ``dryrun`` is ``False``.
557 :param symlink_handling: How to handle symbolic links during sync.
558 :py:attr:`SymlinkHandling.FOLLOW` (default) dereferences symlinks and copies the target's bytes.
559 :py:attr:`SymlinkHandling.SKIP` excludes symlinks from the sync.
560 :py:attr:`SymlinkHandling.PRESERVE` recreates symlinks on the target via :py:meth:`make_symlink`
561 instead of copying bytes (required for round-trip preservation of symlinks).
562 :raises ValueError: If both source_files and patterns are provided.
563 :raises NotImplementedError: If sync operations are not supported (e.g., CompositeStorageClient as target).
564 """
565 return self._delegate.sync_from(
566 source_client,
567 source_path,
568 target_path,
569 delete_unmatched_files,
570 description,
571 num_worker_processes,
572 execution_mode,
573 patterns,
574 preserve_source_attributes,
575 source_files,
576 ignore_hidden,
577 commit_metadata,
578 dryrun,
579 dryrun_output_path,
580 symlink_handling,
581 )
582
[docs]
583 def sync_replicas(
584 self,
585 source_path: str,
586 replica_indices: Sequence[int] | None = None,
587 delete_unmatched_files: bool = False,
588 description: str = "Syncing replica",
589 num_worker_processes: int | None = None,
590 execution_mode: ExecutionMode = ExecutionMode.LOCAL,
591 patterns: PatternList | None = None,
592 ignore_hidden: bool = True,
593 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
594 ) -> None:
595 """
596 Sync files from this client to its replica storage clients.
597
598 :param source_path: The logical path to sync from.
599 :param replica_indices: Specific replica indices to sync to (0-indexed). If None, syncs to all replicas.
600 :param delete_unmatched_files: When set to ``True``, delete files in replicas that don't exist in source.
601 :param description: Description of sync process for logging purposes.
602 :param num_worker_processes: Number of worker processes for parallel sync.
603 :param execution_mode: Execution mode (LOCAL or REMOTE).
604 :param patterns: PatternList for include/exclude filtering. If None, all files are included.
605 :param ignore_hidden: When set to ``True``, ignore hidden files (starting with '.'). Defaults to ``True``.
606 :param symlink_handling: How to handle symbolic links during sync.
607 :py:attr:`SymlinkHandling.FOLLOW` (default) dereferences symlinks and copies the target's bytes.
608 :py:attr:`SymlinkHandling.SKIP` excludes symlinks from the sync.
609 :py:attr:`SymlinkHandling.PRESERVE` recreates symlinks on each replica via
610 :py:meth:`make_symlink` instead of copying bytes.
611 """
612 return self._delegate.sync_replicas(
613 source_path,
614 replica_indices,
615 delete_unmatched_files,
616 description,
617 num_worker_processes,
618 execution_mode,
619 patterns,
620 ignore_hidden,
621 symlink_handling,
622 )
623
[docs]
624 def list(
625 self,
626 path: str = "",
627 start_after: str | None = None,
628 end_at: str | None = None,
629 include_directories: bool = False,
630 include_url_prefix: bool = False,
631 attribute_filter_expression: str | None = None,
632 show_attributes: bool = False,
633 patterns: PatternList | None = None,
634 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
635 ) -> Iterator[ObjectMetadata]:
636 """
637 List objects in the storage provider under the specified path.
638
639 :param path: The directory or file path to list objects under. This should be a
640 complete filesystem path (e.g., "my-bucket/documents/" or "data/2024/").
641 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist.
642 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist.
643 :param include_directories: Whether to include directories in the result. When ``True``, directories are returned alongside objects.
644 :param include_url_prefix: Whether to include the URL prefix ``msc://profile`` in the result.
645 :param attribute_filter_expression: The attribute filter expression to apply to the result.
646 :param show_attributes: Whether to return attributes in the result. WARNING: Depending on implementation, there may be a performance impact if this is set to ``True``.
647 :param patterns: PatternList for include/exclude filtering. If None, all files are included.
648 :param symlink_handling: How to handle symbolic links during listing. Only applicable for POSIX file storage providers.
649 :return: An iterator over ObjectMetadata for matching objects.
650 """
651 return self._delegate.list(
652 path,
653 start_after,
654 end_at,
655 include_directories,
656 include_url_prefix,
657 attribute_filter_expression,
658 show_attributes,
659 patterns=patterns,
660 symlink_handling=symlink_handling,
661 )
662
[docs]
663 def generate_presigned_url(
664 self,
665 path: str,
666 *,
667 method: str = "GET",
668 signer_type: SignerType | None = None,
669 signer_options: dict[str, Any] | None = None,
670 ) -> str:
671 """
672 Generate a pre-signed URL granting temporary access to the object at *path*.
673
674 :param path: The logical path of the object.
675 :param method: The HTTP method the URL should authorise (e.g. ``"GET"``, ``"PUT"``).
676 :param signer_type: The signing backend to use. ``None`` means the provider's native signer.
677 :param signer_options: Backend-specific options forwarded to the signer.
678 :return: A pre-signed URL string.
679 :raises NotImplementedError: If the underlying storage provider does not support presigned URLs.
680 """
681 return self._delegate.generate_presigned_url(
682 path, method=method, signer_type=signer_type, signer_options=signer_options
683 )
684
685 def __getstate__(self) -> dict[str, Any]:
686 """Support for pickling (forward to delegate)."""
687 return self._delegate.__getstate__()
688
689 def __setstate__(self, state: dict[str, Any]) -> None:
690 """Support for unpickling - reconstruct the delegate."""
691 config = state["_config"]
692
693 if config.storage_provider_profiles:
694 self._delegate = CompositeStorageClient.__new__(CompositeStorageClient)
695 self._delegate.__setstate__(state)
696 else:
697 self._delegate = SingleStorageClient.__new__(SingleStorageClient)
698 self._delegate.__setstate__(state)