1# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2# SPDX-License-Identifier: Apache-2.0
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import builtins
17import logging
18import os
19import re
20import threading
21from collections.abc import Callable, Iterator
22from typing import Any
23from urllib.parse import ParseResult, urlparse
24
25from .client import StorageClient
26from .config import RESERVED_POSIX_PROFILE_NAME, SUPPORTED_IMPLICIT_PROFILE_PROTOCOLS, PathMapping, StorageClientConfig
27from .file import ObjectFile, PosixFile
28from .telemetry import Telemetry
29from .types import MSC_PROTOCOL, ExecutionMode, ObjectMetadata, PatternList, SignerType, SymlinkHandling, SyncResult
30
31_TELEMETRY_PROVIDER: Callable[[], Telemetry] | None = None
32_TELEMETRY_PROVIDER_LOCK = threading.Lock()
33_STORAGE_CLIENT_CACHE: dict[str, StorageClient] = {}
34_STORAGE_CLIENT_CACHE_LOCK = threading.Lock()
35_PATH_MAPPING_CACHE: dict[str | None, PathMapping] = {}
36_PATH_MAPPING_CACHE_LOCK = threading.Lock()
37_PROCESS_ID = os.getpid()
38
39logger = logging.getLogger(__name__)
40
41
42def _reinitialize_after_fork() -> None:
43 """
44 Reinitialize module state after fork to ensure fork-safety.
45
46 This function is called automatically after a fork to:
47 1. Clear the storage client cache (cached clients may have invalid state)
48 2. Reinitialize locks (parent's lock state must not be inherited)
49 3. Update process ID tracking
50
51 Note: The telemetry provider is intentionally inherited by child processes,
52 only its lock is reinitialized.
53 """
54 global _STORAGE_CLIENT_CACHE, _STORAGE_CLIENT_CACHE_LOCK # noqa: PLW0602
55 global _PATH_MAPPING_CACHE, _PATH_MAPPING_CACHE_LOCK # noqa: PLW0602
56 global _TELEMETRY_PROVIDER_LOCK
57 global _PROCESS_ID
58
59 _STORAGE_CLIENT_CACHE.clear()
60 _STORAGE_CLIENT_CACHE_LOCK = threading.Lock()
61 _PATH_MAPPING_CACHE.clear()
62 _PATH_MAPPING_CACHE_LOCK = threading.Lock()
63 # we don't need to reset telemetry provider as it is supposed to be a top-level Python function
64 _TELEMETRY_PROVIDER_LOCK = threading.Lock()
65 _PROCESS_ID = os.getpid()
66
67
68def _check_and_reinitialize_if_forked() -> None:
69 """
70 Check if the current process is a fork and reinitialize if needed.
71
72 This provides fork-safety for systems where os.register_at_fork is not available
73 or as a fallback mechanism.
74 """
75 global _PROCESS_ID # noqa: PLW0602
76
77 current_pid = os.getpid()
78 if current_pid != _PROCESS_ID:
79 _reinitialize_after_fork()
80
81
82if hasattr(os, "register_at_fork"):
83 os.register_at_fork(after_in_child=_reinitialize_after_fork)
84
85
[docs]
86def get_telemetry_provider() -> Callable[[], Telemetry] | None:
87 """
88 Get the function used to create :py:class:`Telemetry` instances for storage clients created by shortcuts.
89
90 :return: A function that provides a telemetry instance.
91 """
92 global _TELEMETRY_PROVIDER # noqa: PLW0602
93
94 return _TELEMETRY_PROVIDER
95
96
[docs]
97def set_telemetry_provider(telemetry_provider: Callable[[], Telemetry] | None) -> None:
98 """
99 Set the function used to create :py:class:`Telemetry` instances for storage clients created by shortcuts.
100
101 :param telemetry_provider: A function that provides a telemetry instance. The function must be defined at the top level of a module to work with pickling.
102 """
103 global _TELEMETRY_PROVIDER
104 global _TELEMETRY_PROVIDER_LOCK # noqa: PLW0602
105
106 with _TELEMETRY_PROVIDER_LOCK:
107 _TELEMETRY_PROVIDER = telemetry_provider
108
109
110def _build_full_path(original_url: str, pr: ParseResult) -> str:
111 """
112 Helper function to construct the full path from a parsed URL, including query and fragment.
113
114 :param original_url: The original URL before parsing
115 :param pr: The parsed URL result from urlparse
116 :return: The complete path including query and fragment if present
117 """
118 path = pr.path
119 if pr.query:
120 path += "?" + pr.query
121 elif original_url.endswith("?"):
122 path += "?" # handle the glob pattern that has a trailing question mark
123 if pr.fragment:
124 path += "#" + pr.fragment
125 return path
126
127
128def _resolve_msc_url(url: str) -> tuple[str, str]:
129 """
130 Resolve an MSC URL to a profile name and path.
131
132 :param url: The MSC URL to resolve (msc://profile/path)
133 :return: A tuple of (profile_name, path)
134 """
135 pr = urlparse(url)
136 profile = pr.netloc
137 # Normalize only the object path so the msc:// scheme separator and profile stay intact.
138 pr = pr._replace(path=re.sub(r"/+", "/", pr.path))
139 path = _build_full_path(url, pr)
140 path = path.removeprefix("/")
141 return profile, path
142
143
144def _read_cached_path_mapping() -> PathMapping:
145 """
146 Read path mapping once per ``MSC_CONFIG`` value for shortcut URL resolution.
147
148 Path mapping checks happen on every non-MSC shortcut call, including POSIX paths. Caching here keeps that hot path
149 from repeatedly loading and validating the full MSC config while preserving ``StorageClientConfig.read_path_mapping``
150 behavior for direct callers.
151 """
152 cache_key = os.getenv("MSC_CONFIG", None)
153 if cache_key in _PATH_MAPPING_CACHE:
154 return _PATH_MAPPING_CACHE[cache_key]
155
156 with _PATH_MAPPING_CACHE_LOCK:
157 if cache_key in _PATH_MAPPING_CACHE:
158 return _PATH_MAPPING_CACHE[cache_key]
159
160 path_mapping = StorageClientConfig.read_path_mapping()
161 _PATH_MAPPING_CACHE[cache_key] = path_mapping
162 return path_mapping
163
164
165def _resolve_non_msc_url(url: str) -> tuple[str, str]:
166 """
167 Resolve a non-MSC URL to a profile name and path.
168
169 Resolution process:
170 1. First check if MSC config exists
171 2. If config exists, check for possible path mapping
172 3. If no mapping is found, fall back to the reserved POSIX profile (``__filesystem__``) for file paths or create an implicit profile based on URL
173
174 :param url: The non-MSC URL to resolve
175 :return: A tuple of (profile_name, path)
176 """
177 # Check if we have a valid path mapping, if so check if there is a matching mapping
178 path_mapping = _read_cached_path_mapping()
179 if path_mapping:
180 # Look for a matching mapping
181 possible_mapping = path_mapping.find_mapping(url)
182 if possible_mapping:
183 return possible_mapping # return the profile name and path
184
185 # For file paths, use the default POSIX profile
186 if url.startswith("file://"):
187 pr = urlparse(url)
188 return RESERVED_POSIX_PROFILE_NAME, _build_full_path(url, pr)
189 elif url.startswith("/"):
190 url = os.path.normpath(url)
191 return RESERVED_POSIX_PROFILE_NAME, url
192
193 # For other URL protocol, create an implicit profile name
194 pr = urlparse(url)
195 protocol = pr.scheme.lower()
196
197 # Translate relative paths to absolute paths
198 if not protocol:
199 return RESERVED_POSIX_PROFILE_NAME, os.path.realpath(url)
200
201 # Validate the protocol is supported
202 if protocol not in SUPPORTED_IMPLICIT_PROFILE_PROTOCOLS:
203 supported_protocols = ", ".join([f"{p}://" for p in SUPPORTED_IMPLICIT_PROFILE_PROTOCOLS])
204 raise ValueError(
205 f'Unknown URL "{url}", expecting "{MSC_PROTOCOL}" or a supported protocol ({supported_protocols}) or a POSIX path'
206 )
207
208 # Build the implicit profile name using the format _protocol-bucket
209 bucket = pr.netloc
210 if not bucket:
211 raise ValueError(f'Invalid URL "{url}", bucket name is required for {protocol}:// URLs')
212
213 profile_name = f"_{protocol}-{bucket}"
214
215 # Return normalized path with leading slash removed
216 path = pr.path
217 path = path.removeprefix("/")
218
219 return profile_name, path
220
221
[docs]
222def resolve_storage_client(url: str) -> tuple[StorageClient, str]:
223 """
224 Build and return a :py:class:`multistorageclient.StorageClient` instance based on the provided URL or path.
225
226 This function parses the given URL or path and determines the appropriate storage profile and path.
227 It supports URLs with the protocol ``msc://``, as well as POSIX paths or ``file://`` URLs for local file
228 system access. If the profile has already been instantiated, it returns the cached client. Otherwise,
229 it creates a new :py:class:`StorageClient` and caches it.
230
231 The function also supports implicit profiles for non-MSC URLs. When a non-MSC URL is provided (like s3://,
232 gs://, ais://, file://), MSC will infer the storage provider based on the URL protocol and create an implicit
233 profile with the naming convention "_protocol-bucket" (e.g., "_s3-bucket1", "_gs-bucket1").
234
235 Path mapping defined in the MSC configuration are also applied before creating implicit profiles.
236 This allows for explicit mappings between source paths and destination MSC profiles.
237
238 This function is fork-safe: after a fork, the cache is automatically cleared and new client instances
239 are created in the child process to avoid sharing stale connections or file descriptors.
240
241 :param url: The storage location, which can be:
242 - A URL in the format ``msc://profile/path`` for object storage.
243 - A local file system path (absolute POSIX path) or a ``file://`` URL.
244 - A non-MSC URL with a supported protocol (s3://, gs://, ais://).
245
246 :return: A tuple containing the :py:class:`multistorageclient.StorageClient` instance and the parsed path.
247
248 :raises ValueError: If the URL's protocol is neither ``msc`` nor a valid local file system path
249 or a supported non-MSC protocol.
250 """
251 global _STORAGE_CLIENT_CACHE # noqa: PLW0602
252 global _STORAGE_CLIENT_CACHE_LOCK # noqa: PLW0602
253
254 _check_and_reinitialize_if_forked()
255
256 # Normalize the path for msc:/ prefix due to pathlib.Path('msc://')
257 if url.startswith("msc:/") and not url.startswith("msc://"):
258 url = url.replace("msc:/", "msc://")
259
260 # Resolve the URL to a profile name and path
261 profile, path = _resolve_msc_url(url) if url.startswith(MSC_PROTOCOL) else _resolve_non_msc_url(url)
262
263 # Check if the profile has already been instantiated
264 if profile in _STORAGE_CLIENT_CACHE:
265 return _STORAGE_CLIENT_CACHE[profile], path
266
267 # Create a new StorageClient instance and cache it
268 with _STORAGE_CLIENT_CACHE_LOCK:
269 if profile in _STORAGE_CLIENT_CACHE:
270 return _STORAGE_CLIENT_CACHE[profile], path
271 else:
272 client = StorageClient(
273 config=StorageClientConfig.from_file(profile=profile, telemetry_provider=get_telemetry_provider())
274 )
275 _STORAGE_CLIENT_CACHE[profile] = client
276
277 return client, path
278
279
[docs]
280def open(url: str, mode: str = "rb", **kwargs: Any) -> PosixFile | ObjectFile:
281 """
282 Open a file at the given URL using the specified mode.
283
284 The function utilizes the :py:class:`multistorageclient.StorageClient` to open a file at the provided path.
285 The URL is parsed, and the corresponding :py:class:`multistorageclient.StorageClient` is retrieved or built.
286
287 :param url: The URL of the file to open. (example: ``msc://profile/prefix/dataset.tar``)
288 :param mode: The file mode to open the file in.
289
290 :return: A file-like object that allows interaction with the file.
291
292 :raises ValueError: If the URL's protocol does not match the expected protocol ``msc``.
293 """
294 client, path = resolve_storage_client(url)
295 return client.open(path, mode, **kwargs)
296
297
[docs]
298def glob(pattern: str, attribute_filter_expression: str | None = None) -> builtins.list[str]:
299 """
300 Return a list of files matching a pattern.
301
302 This function supports glob-style patterns for matching multiple files within a storage system. The pattern is
303 parsed, and the associated :py:class:`multistorageclient.StorageClient` is used to retrieve the
304 list of matching files.
305
306 :param pattern: The glob-style pattern to match files. (example: ``msc://profile/prefix/**/*.tar``)
307 :param attribute_filter_expression: The attribute filter expression to apply to the result.
308
309 :return: A list of file paths matching the pattern.
310
311 :raises ValueError: If the URL's protocol does not match the expected protocol ``msc``.
312 """
313 client, path = resolve_storage_client(pattern)
314 if not pattern.startswith(MSC_PROTOCOL) and client.profile == RESERVED_POSIX_PROFILE_NAME:
315 return client.glob(path, include_url_prefix=False, attribute_filter_expression=attribute_filter_expression)
316 else:
317 return client.glob(path, include_url_prefix=True, attribute_filter_expression=attribute_filter_expression)
318
319
[docs]
320def upload_file(url: str, local_path: str, attributes: dict[str, Any] | None = None) -> None:
321 """
322 Upload a file to the given URL from a local path.
323
324 The function utilizes the :py:class:`multistorageclient.StorageClient` to upload a file (object) to the
325 provided path. The URL is parsed, and the corresponding :py:class:`multistorageclient.StorageClient`
326 is retrieved or built.
327
328 :param url: The URL of the file. (example: ``msc://profile/prefix/dataset.tar``)
329 :param local_path: The local path of the file.
330
331 :raises ValueError: If the URL's protocol does not match the expected protocol ``msc``.
332 """
333 client, path = resolve_storage_client(url)
334 return client.upload_file(remote_path=path, local_path=local_path, attributes=attributes)
335
336
[docs]
337def download_file(url: str, local_path: str) -> None:
338 """
339 Download a file in a given remote_path to a local path
340
341 The function utilizes the :py:class:`multistorageclient.StorageClient` to download a file (object) at the
342 provided path. The URL is parsed, and the corresponding :py:class:`multistorageclient.StorageClient`
343 is retrieved or built.
344
345 :param url: The URL of the file to download. (example: ``msc://profile/prefix/dataset.tar``)
346 :param local_path: The local path where the file should be downloaded.
347
348 :raises ValueError: If the URL's protocol does not match the expected protocol ``msc``.
349 """
350 client, path = resolve_storage_client(url)
351 return client.download_file(remote_path=path, local_path=local_path)
352
353
[docs]
354def is_empty(url: str) -> bool:
355 """
356 Checks whether the specified URL contains any objects.
357
358 :param url: The URL to check, typically pointing to a storage location.
359 :return: ``True`` if there are no objects/files under this URL, ``False`` otherwise.
360
361 :raises ValueError: If the URL's protocol does not match the expected protocol ``msc``.
362 """
363 client, path = resolve_storage_client(url)
364 return client.is_empty(path)
365
366
[docs]
367def is_file(url: str) -> bool:
368 """
369 Checks whether the specified url points to a file (rather than a directory or folder).
370
371 The function utilizes the :py:class:`multistorageclient.StorageClient` to check if a file (object) exists
372 at the provided path. The URL is parsed, and the corresponding :py:class:`multistorageclient.StorageClient`
373 is retrieved or built.
374
375 :param url: The URL to check the existence of a file. (example: ``msc://profile/prefix/dataset.tar``)
376 """
377 client, path = resolve_storage_client(url)
378 return client.is_file(path=path)
379
380
[docs]
381def sync(
382 source_url: str,
383 target_url: str,
384 delete_unmatched_files: bool = False,
385 execution_mode: ExecutionMode = ExecutionMode.LOCAL,
386 patterns: PatternList | None = None,
387 preserve_source_attributes: bool = False,
388 ignore_hidden: bool = True,
389 dryrun: bool = False,
390 dryrun_output_path: str | None = None,
391 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
392) -> SyncResult:
393 """
394 Syncs files from the source storage to the target storage.
395
396 :param source_url: The URL for the source storage.
397 :param target_url: The URL for the target storage.
398 :param delete_unmatched_files: Whether to delete files at the target that are not present at the source.
399 :param execution_mode: The execution mode to use. Currently supports "local" and "ray".
400 :param patterns: PatternList for include/exclude filtering. If None, all files are included.
401 :param preserve_source_attributes: Whether to preserve source file metadata attributes during synchronization.
402 When False (default), only file content is copied. When True, custom metadata attributes are also preserved.
403
404 .. warning::
405 **Performance Impact**: When enabled without a ``metadata_provider`` configured, this will make a HEAD
406 request for each object to retrieve attributes, which can significantly impact performance on large-scale
407 sync operations. For production use at scale, configure a ``metadata_provider`` in your storage profile.
408 :param ignore_hidden: Whether to ignore hidden files and directories (starting with dot). Default is True.
409 :param dryrun: If True, only enumerate and compare objects without performing any copy/delete operations.
410 The returned :py:class:`SyncResult` will include a :py:class:`DryrunResult` with paths to JSONL files.
411 :param dryrun_output_path: Directory to write dryrun JSONL files into. If None (default), a temporary
412 directory is created automatically. Ignored when dryrun is False.
413 :param symlink_handling: How to handle symbolic links during sync.
414 :py:attr:`SymlinkHandling.FOLLOW` (default) dereferences symlinks and copies the target's bytes.
415 :py:attr:`SymlinkHandling.SKIP` excludes symlinks from the sync.
416 :py:attr:`SymlinkHandling.PRESERVE` recreates symlinks on the target via
417 :py:meth:`AbstractStorageClient.make_symlink` instead of copying bytes (required for
418 round-trip preservation of symlinks).
419 """
420 source_client, source_path = resolve_storage_client(source_url)
421 target_client, target_path = resolve_storage_client(target_url)
422 return target_client.sync_from(
423 source_client,
424 source_path,
425 target_path,
426 delete_unmatched_files,
427 execution_mode=execution_mode,
428 patterns=patterns,
429 preserve_source_attributes=preserve_source_attributes,
430 ignore_hidden=ignore_hidden,
431 dryrun=dryrun,
432 dryrun_output_path=dryrun_output_path,
433 symlink_handling=symlink_handling,
434 )
435
436
[docs]
437def sync_replicas(
438 source_url: str,
439 replica_indices: builtins.list[int] | None = None,
440 delete_unmatched_files: bool = False,
441 execution_mode: ExecutionMode = ExecutionMode.LOCAL,
442 patterns: PatternList | None = None,
443 ignore_hidden: bool = True,
444 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
445) -> None:
446 """
447 Syncs files from the source storage to all the replicas.
448
449 :param source_url: The URL for the source storage.
450 :param replica_indices: Specify the indices of the replicas to sync to. If not provided, all replicas will be synced. Index starts from 0.
451 :param delete_unmatched_files: Whether to delete files at the replicas that are not present at the source.
452 :param execution_mode: The execution mode to use. Currently supports "local" and "ray".
453 :param patterns: PatternList for include/exclude filtering. If None, all files are included.
454 :param ignore_hidden: Whether to ignore hidden files and directories (starting with dot). Default is True.
455 :param symlink_handling: How to handle symbolic links during sync.
456 :py:attr:`SymlinkHandling.FOLLOW` (default) dereferences symlinks and copies the target's bytes.
457 :py:attr:`SymlinkHandling.SKIP` excludes symlinks from the sync.
458 :py:attr:`SymlinkHandling.PRESERVE` recreates symlinks on each replica via
459 :py:meth:`AbstractStorageClient.make_symlink` instead of copying bytes.
460 """
461 source_client, source_path = resolve_storage_client(source_url)
462 source_client.sync_replicas(
463 source_path,
464 replica_indices=replica_indices,
465 delete_unmatched_files=delete_unmatched_files,
466 execution_mode=execution_mode,
467 patterns=patterns,
468 ignore_hidden=ignore_hidden,
469 symlink_handling=symlink_handling,
470 )
471
472
[docs]
473def list(
474 url: str,
475 start_after: str | None = None,
476 end_at: str | None = None,
477 include_directories: bool = False,
478 attribute_filter_expression: str | None = None,
479 show_attributes: bool = False,
480 patterns: PatternList | None = None,
481 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
482) -> Iterator[ObjectMetadata]:
483 """
484 Lists the contents of the specified URL prefix.
485
486 This function retrieves the corresponding :py:class:`multistorageclient.StorageClient`
487 for the given URL and returns an iterator of objects (files or directories) stored under the provided prefix.
488
489 :param url: The prefix to list objects under.
490 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist.
491 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist.
492 :param include_directories: Whether to include directories in the result. When True, directories are returned alongside objects.
493 :param attribute_filter_expression: The attribute filter expression to apply to the result.
494 :param show_attributes: Whether to return attributes in the result.
495 :param patterns: PatternList for include/exclude filtering. If None, all files are included.
496 :param symlink_handling: How to handle symbolic links. Only applicable for POSIX file storage.
497 :return: An iterator of :py:class:`ObjectMetadata` objects representing the files (and optionally directories)
498 accessible under the specified URL prefix. The returned keys will always be prefixed with msc://.
499 """
500 client, path = resolve_storage_client(url)
501 return client.list(
502 path=path,
503 start_after=start_after,
504 end_at=end_at,
505 include_directories=include_directories,
506 include_url_prefix=True,
507 attribute_filter_expression=attribute_filter_expression,
508 show_attributes=show_attributes,
509 patterns=patterns,
510 symlink_handling=symlink_handling,
511 )
512
513
[docs]
514def list_recursive(
515 url: str,
516 start_after: str | None = None,
517 end_at: str | None = None,
518 max_workers: int = 32,
519 look_ahead: int = 2,
520 patterns: PatternList | None = None,
521 symlink_handling: SymlinkHandling = SymlinkHandling.FOLLOW,
522) -> Iterator[ObjectMetadata]:
523 """
524 Lists files recursively under the specified URL.
525
526 This function retrieves the corresponding :py:class:`multistorageclient.StorageClient`
527 for the given URL and returns an iterator of files under the provided path.
528
529 :param url: The path to list objects under.
530 :param start_after: The key to start after (i.e. exclusive). An object with this key doesn't have to exist.
531 :param end_at: The key to end at (i.e. inclusive). An object with this key doesn't have to exist.
532 :param max_workers: Maximum concurrent workers for provider-level recursive listing.
533 :param look_ahead: Prefixes to buffer per worker for provider-level recursive listing.
534 :param patterns: PatternList for include/exclude filtering. If None, all files are included.
535 :param symlink_handling: How to handle symbolic links during listing.
536 :return: An iterator of :py:class:`ObjectMetadata` objects representing files accessible under the specified URL path.
537 The returned keys use the same URL-prefix behavior as :py:meth:`multistorageclient.list`.
538 """
539 client, path = resolve_storage_client(url)
540 return client.list_recursive(
541 path=path,
542 start_after=start_after,
543 end_at=end_at,
544 max_workers=max_workers,
545 look_ahead=look_ahead,
546 include_url_prefix=True,
547 patterns=patterns,
548 symlink_handling=symlink_handling,
549 )
550
551
[docs]
552def write(url: str, body: bytes, attributes: dict[str, Any] | None = None) -> None:
553 """
554 Writes an object to the storage provider at the specified path.
555
556 :param url: The path where the object should be written.
557 :param body: The content to write to the object.
558 """
559 client, path = resolve_storage_client(url)
560 client.write(path=path, body=body, attributes=attributes)
561
562
[docs]
563def make_symlink(url: str, target_url: str) -> None:
564 """
565 Creates a symbolic link at ``url`` pointing to ``target_url``.
566
567 Both URLs must resolve to the same storage profile.
568
569 :param url: The URL where the symlink will be created.
570 :param target_url: The URL of the target that the symlink points to.
571 :raises ValueError: If the two URLs resolve to different storage profiles.
572 """
573 client, path = resolve_storage_client(url)
574 target_client, target_path = resolve_storage_client(target_url)
575 if client is not target_client:
576 raise ValueError("Cannot create cross-profile symlink: url and target_url must belong to the same profile.")
577 client.make_symlink(path=path, target=target_path)
578
579
[docs]
580def delete(url: str, recursive: bool = False) -> None:
581 """
582 Deletes the specified object(s) from the storage provider.
583
584 This function retrieves the corresponding :py:class:`multistorageclient.StorageClient`
585 for the given URL and deletes the object(s) at the specified path.
586
587 :param url: The URL of the object to delete. (example: ``msc://profile/prefix/file.txt``)
588 :param recursive: Whether to delete objects in the path recursively.
589 """
590 client, path = resolve_storage_client(url)
591 client.delete(path, recursive=recursive)
592
593
[docs]
594def info(url: str) -> ObjectMetadata:
595 """
596 Retrieves metadata or information about an object stored at the specified path.
597
598 :param url: The URL of the object to retrieve information about. (example: ``msc://profile/prefix/file.txt``)
599
600 :return: An :py:class:`ObjectMetadata` object representing the object's metadata.
601 """
602 client, path = resolve_storage_client(url)
603 return client.info(path)
604
605
614
615
[docs]
616def generate_presigned_url(
617 url: str,
618 *,
619 method: str = "GET",
620 signer_type: SignerType | None = None,
621 signer_options: dict[str, Any] | None = None,
622) -> str:
623 """
624 Generate a pre-signed URL granting temporary access to the object at *url*.
625
626 :param url: The storage URL. (example: ``msc://profile/prefix/file.bin``)
627 :param method: The HTTP method the URL should authorise (e.g. ``"GET"``, ``"PUT"``).
628 :param signer_type: The signing backend to use. ``None`` means the provider's native signer.
629 :param signer_options: Backend-specific options forwarded to the signer.
630 :return: A pre-signed URL string.
631 """
632 client, path = resolve_storage_client(url)
633 return client.generate_presigned_url(path, method=method, signer_type=signer_type, signer_options=signer_options)