This is page 54 of 74. Use http://codebase.md/apache/opendal?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .asf.yaml
├── .config
│ └── nextest.toml
├── .devcontainer
│ ├── devcontainer.json
│ └── post_create.sh
├── .editorconfig
├── .env.example
├── .gitattributes
├── .github
│ ├── actions
│ │ ├── fuzz_test
│ │ │ └── action.yaml
│ │ ├── setup
│ │ │ └── action.yaml
│ │ ├── setup-hadoop
│ │ │ └── action.yaml
│ │ ├── setup-ocaml
│ │ │ └── action.yaml
│ │ ├── test_behavior_binding_c
│ │ │ └── action.yaml
│ │ ├── test_behavior_binding_cpp
│ │ │ └── action.yaml
│ │ ├── test_behavior_binding_go
│ │ │ └── action.yaml
│ │ ├── test_behavior_binding_java
│ │ │ └── action.yaml
│ │ ├── test_behavior_binding_nodejs
│ │ │ └── action.yaml
│ │ ├── test_behavior_binding_python
│ │ │ └── action.yaml
│ │ ├── test_behavior_core
│ │ │ └── action.yaml
│ │ └── test_behavior_integration_object_store
│ │ └── action.yml
│ ├── CODEOWNERS
│ ├── dependabot.yml
│ ├── ISSUE_TEMPLATE
│ │ ├── 1-bug-report.yml
│ │ ├── 2-feature-request.yml
│ │ ├── 3-new-release.md
│ │ └── config.yml
│ ├── pull_request_template.md
│ ├── release.yml
│ ├── scripts
│ │ ├── test_behavior
│ │ │ ├── __init__.py
│ │ │ ├── plan.py
│ │ │ └── test_plan.py
│ │ ├── test_go_binding
│ │ │ ├── generate_test_scheme.py
│ │ │ └── matrix.yaml
│ │ └── weekly_update
│ │ ├── .gitignore
│ │ ├── .python-version
│ │ ├── main.py
│ │ ├── pyproject.toml
│ │ ├── README.md
│ │ └── uv.lock
│ ├── services
│ │ ├── aliyun_drive
│ │ │ └── aliyun_drive
│ │ │ └── disable_action.yml
│ │ ├── alluxio
│ │ │ └── alluxio
│ │ │ └── action.yml
│ │ ├── azblob
│ │ │ ├── azure_azblob
│ │ │ │ └── action.yml
│ │ │ └── azurite_azblob
│ │ │ └── action.yml
│ │ ├── azdls
│ │ │ └── azdls
│ │ │ └── action.yml
│ │ ├── azfile
│ │ │ └── azfile
│ │ │ └── action.yml
│ │ ├── b2
│ │ │ └── b2
│ │ │ └── action.yml
│ │ ├── cacache
│ │ │ └── cacache
│ │ │ └── action.yml
│ │ ├── compfs
│ │ │ └── compfs
│ │ │ └── action.yml
│ │ ├── cos
│ │ │ └── cos
│ │ │ └── action.yml
│ │ ├── dashmap
│ │ │ └── dashmap
│ │ │ └── action.yml
│ │ ├── dropbox
│ │ │ └── dropbox
│ │ │ └── disable_action.yml
│ │ ├── etcd
│ │ │ ├── etcd
│ │ │ │ └── action.yml
│ │ │ ├── etcd-cluster
│ │ │ │ └── action.yml
│ │ │ └── etcd-tls
│ │ │ └── action.yml
│ │ ├── fs
│ │ │ └── local_fs
│ │ │ └── action.yml
│ │ ├── ftp
│ │ │ └── vsftpd
│ │ │ └── disable_action.yml
│ │ ├── gcs
│ │ │ ├── gcs
│ │ │ │ └── action.yml
│ │ │ └── gcs_with_default_storage_class
│ │ │ └── action.yml
│ │ ├── gdrive
│ │ │ └── gdrive
│ │ │ └── action.yml
│ │ ├── gridfs
│ │ │ ├── gridfs
│ │ │ │ └── action.yml
│ │ │ └── gridfs_with_basic_auth
│ │ │ └── action.yml
│ │ ├── hdfs
│ │ │ ├── hdfs_cluster
│ │ │ │ └── action.yml
│ │ │ ├── hdfs_cluster_with_atomic_write_dir
│ │ │ │ └── action.yml
│ │ │ ├── hdfs_default
│ │ │ │ └── action.yml
│ │ │ ├── hdfs_default_gcs
│ │ │ │ └── action.yml
│ │ │ ├── hdfs_default_on_azurite_azblob
│ │ │ │ └── action.yml
│ │ │ ├── hdfs_default_on_minio_s3
│ │ │ │ └── action.yml
│ │ │ └── hdfs_default_with_atomic_write_dir
│ │ │ └── action.yml
│ │ ├── hdfs_native
│ │ │ └── hdfs_native_cluster
│ │ │ └── action.yml
│ │ ├── http
│ │ │ ├── caddy
│ │ │ │ └── action.yml
│ │ │ └── nginx
│ │ │ └── action.yml
│ │ ├── huggingface
│ │ │ └── huggingface
│ │ │ └── action.yml
│ │ ├── koofr
│ │ │ └── koofr
│ │ │ └── disable_action.yml
│ │ ├── memcached
│ │ │ ├── memcached
│ │ │ │ └── action.yml
│ │ │ └── memcached_with_auth
│ │ │ └── action.yml
│ │ ├── memory
│ │ │ └── memory
│ │ │ └── action.yml
│ │ ├── mini_moka
│ │ │ └── mini_moka
│ │ │ └── action.yml
│ │ ├── moka
│ │ │ └── moka
│ │ │ └── action.yml
│ │ ├── mongodb
│ │ │ ├── mongodb_with_basic_auth
│ │ │ │ └── action.yml
│ │ │ └── mongodb_with_no_auth
│ │ │ └── action.yml
│ │ ├── monoiofs
│ │ │ └── monoiofs
│ │ │ └── action.yml
│ │ ├── mysql
│ │ │ └── mysql
│ │ │ └── action.yml
│ │ ├── oss
│ │ │ ├── oss
│ │ │ │ └── action.yml
│ │ │ └── oss_with_versioning
│ │ │ └── action.yml
│ │ ├── persy
│ │ │ └── persy
│ │ │ └── action.yml
│ │ ├── postgresql
│ │ │ └── postgresql
│ │ │ └── action.yml
│ │ ├── redb
│ │ │ └── redb
│ │ │ └── action.yml
│ │ ├── redis
│ │ │ ├── dragonfly
│ │ │ │ └── action.yml
│ │ │ ├── kvrocks
│ │ │ │ └── action.yml
│ │ │ ├── redis
│ │ │ │ └── action.yml
│ │ │ ├── redis_tls
│ │ │ │ └── action.yml
│ │ │ ├── redis_with_cluster
│ │ │ │ └── action.yml
│ │ │ └── redis_with_cluster_tls
│ │ │ └── action.yml
│ │ ├── rocksdb
│ │ │ └── rocksdb
│ │ │ └── action.yml
│ │ ├── s3
│ │ │ ├── 0_minio_s3
│ │ │ │ └── action.yml
│ │ │ ├── aws_s3
│ │ │ │ └── action.yml
│ │ │ ├── aws_s3_with_list_objects_v1
│ │ │ │ └── action.yml
│ │ │ ├── aws_s3_with_sse_c
│ │ │ │ └── action.yml
│ │ │ ├── aws_s3_with_versioning
│ │ │ │ └── action.yml
│ │ │ ├── aws_s3_with_virtual_host
│ │ │ │ └── action.yml
│ │ │ ├── ceph_radios_s3_with_versioning
│ │ │ │ └── disable_action.yml
│ │ │ ├── ceph_rados_s3
│ │ │ │ └── disable_action.yml
│ │ │ ├── minio_s3_with_anonymous
│ │ │ │ └── action.yml
│ │ │ ├── minio_s3_with_list_objects_v1
│ │ │ │ └── action.yml
│ │ │ ├── minio_s3_with_versioning
│ │ │ │ └── action.yml
│ │ │ └── r2
│ │ │ └── disabled_action.yml
│ │ ├── seafile
│ │ │ └── seafile
│ │ │ └── action.yml
│ │ ├── sftp
│ │ │ ├── sftp
│ │ │ │ └── action.yml
│ │ │ └── sftp_with_default_root
│ │ │ └── action.yml
│ │ ├── sled
│ │ │ ├── sled
│ │ │ │ └── action.yml
│ │ │ └── sled_with_tree
│ │ │ └── action.yml
│ │ ├── sqlite
│ │ │ └── sqlite
│ │ │ └── action.yml
│ │ ├── swift
│ │ │ ├── ceph_rados_swift
│ │ │ │ └── action.yml
│ │ │ └── swift
│ │ │ └── action.yml
│ │ ├── tikv
│ │ │ └── tikv
│ │ │ └── disable_action.yml
│ │ ├── webdav
│ │ │ ├── 0_nginx
│ │ │ │ └── action.yml
│ │ │ ├── jfrog
│ │ │ │ └── disabled_action.yml
│ │ │ ├── nextcloud
│ │ │ │ └── action.yml
│ │ │ ├── nginx_with_empty_password
│ │ │ │ └── action.yml
│ │ │ ├── nginx_with_password
│ │ │ │ └── action.yml
│ │ │ ├── nginx_with_redirect
│ │ │ │ └── action.yml
│ │ │ └── owncloud
│ │ │ └── action.yml
│ │ └── webhdfs
│ │ ├── webhdfs
│ │ │ └── action.yml
│ │ ├── webhdfs_with_list_batch_disabled
│ │ │ └── action.yml
│ │ └── webhdfs_with_user_name
│ │ └── action.yml
│ └── workflows
│ ├── ci_bindings_c.yml
│ ├── ci_bindings_cpp.yml
│ ├── ci_bindings_d.yml
│ ├── ci_bindings_dart.yml
│ ├── ci_bindings_dotnet.yml
│ ├── ci_bindings_go.yml
│ ├── ci_bindings_haskell.yml
│ ├── ci_bindings_java.yml
│ ├── ci_bindings_lua.yml
│ ├── ci_bindings_nodejs.yml
│ ├── ci_bindings_ocaml.yml
│ ├── ci_bindings_php.yml
│ ├── ci_bindings_python.yml
│ ├── ci_bindings_ruby.yml
│ ├── ci_bindings_swift.yml
│ ├── ci_bindings_zig.yml
│ ├── ci_check.yml
│ ├── ci_core.yml
│ ├── ci_integration_dav_server.yml
│ ├── ci_integration_object_store.yml
│ ├── ci_integration_parquet.yml
│ ├── ci_integration_spring.yml
│ ├── ci_integration_unftp_sbe.yml
│ ├── ci_odev.yml
│ ├── ci_weekly_update.yml
│ ├── discussion-thread-link.yml
│ ├── docs.yml
│ ├── full-ci-promote.yml
│ ├── release_dart.yml
│ ├── release_java.yml
│ ├── release_nodejs.yml
│ ├── release_python.yml
│ ├── release_ruby.yml
│ ├── release_rust.yml
│ ├── service_test_ghac.yml
│ ├── test_behavior_binding_c.yml
│ ├── test_behavior_binding_cpp.yml
│ ├── test_behavior_binding_go.yml
│ ├── test_behavior_binding_java.yml
│ ├── test_behavior_binding_nodejs.yml
│ ├── test_behavior_binding_python.yml
│ ├── test_behavior_core.yml
│ ├── test_behavior_integration_object_store.yml
│ ├── test_behavior.yml
│ ├── test_edge.yml
│ └── test_fuzz.yml
├── .gitignore
├── .taplo.toml
├── .typos.toml
├── .vscode
│ └── settings.json
├── .yamlfmt
├── AGENTS.md
├── bindings
│ ├── java
│ │ ├── .cargo
│ │ │ └── config.toml
│ │ ├── .gitignore
│ │ ├── .mvn
│ │ │ └── wrapper
│ │ │ └── maven-wrapper.properties
│ │ ├── Cargo.toml
│ │ ├── DEPENDENCIES.md
│ │ ├── DEPENDENCIES.rust.tsv
│ │ ├── mvnw
│ │ ├── mvnw.cmd
│ │ ├── pom.xml
│ │ ├── README.md
│ │ ├── src
│ │ │ ├── async_operator.rs
│ │ │ ├── convert.rs
│ │ │ ├── error.rs
│ │ │ ├── executor.rs
│ │ │ ├── layer.rs
│ │ │ ├── lib.rs
│ │ │ ├── main
│ │ │ │ ├── java
│ │ │ │ │ └── org
│ │ │ │ │ └── apache
│ │ │ │ │ └── opendal
│ │ │ │ │ ├── AsyncExecutor.java
│ │ │ │ │ ├── AsyncOperator.java
│ │ │ │ │ ├── Capability.java
│ │ │ │ │ ├── Entry.java
│ │ │ │ │ ├── Environment.java
│ │ │ │ │ ├── layer
│ │ │ │ │ │ ├── ConcurrentLimitLayer.java
│ │ │ │ │ │ ├── package-info.java
│ │ │ │ │ │ └── RetryLayer.java
│ │ │ │ │ ├── Layer.java
│ │ │ │ │ ├── ListOptions.java
│ │ │ │ │ ├── Metadata.java
│ │ │ │ │ ├── NativeLibrary.java
│ │ │ │ │ ├── NativeObject.java
│ │ │ │ │ ├── OpenDAL.java
│ │ │ │ │ ├── OpenDALException.java
│ │ │ │ │ ├── Operator.java
│ │ │ │ │ ├── OperatorInfo.java
│ │ │ │ │ ├── OperatorInputStream.java
│ │ │ │ │ ├── OperatorOutputStream.java
│ │ │ │ │ ├── package-info.java
│ │ │ │ │ ├── PresignedRequest.java
│ │ │ │ │ ├── ReadOptions.java
│ │ │ │ │ ├── ServiceConfig.java
│ │ │ │ │ ├── StatOptions.java
│ │ │ │ │ └── WriteOptions.java
│ │ │ │ └── resources
│ │ │ │ ├── bindings.properties
│ │ │ │ └── META-INF
│ │ │ │ └── NOTICE
│ │ │ ├── operator_input_stream.rs
│ │ │ ├── operator_output_stream.rs
│ │ │ ├── operator.rs
│ │ │ ├── test
│ │ │ │ └── java
│ │ │ │ └── org
│ │ │ │ └── apache
│ │ │ │ └── opendal
│ │ │ │ └── test
│ │ │ │ ├── AsyncExecutorTest.java
│ │ │ │ ├── behavior
│ │ │ │ │ ├── AsyncCopyTest.java
│ │ │ │ │ ├── AsyncCreateDirTest.java
│ │ │ │ │ ├── AsyncListTest.java
│ │ │ │ │ ├── AsyncPresignTest.java
│ │ │ │ │ ├── AsyncReadOnlyTest.java
│ │ │ │ │ ├── AsyncRenameTest.java
│ │ │ │ │ ├── AsyncStatOptionsTest.java
│ │ │ │ │ ├── AsyncWriteOptionsTest.java
│ │ │ │ │ ├── AsyncWriteTest.java
│ │ │ │ │ ├── BehaviorExtension.java
│ │ │ │ │ ├── BehaviorTestBase.java
│ │ │ │ │ ├── BlockingCopyTest.java
│ │ │ │ │ ├── BlockingCreateDirTest.java
│ │ │ │ │ ├── BlockingListTest.java
│ │ │ │ │ ├── BlockingReadOnlyTest.java
│ │ │ │ │ ├── BlockingRenameTest.java
│ │ │ │ │ ├── BlockingStatOptionsTest.java
│ │ │ │ │ ├── BlockingWriteOptionTest.java
│ │ │ │ │ ├── BlockingWriteTest.java
│ │ │ │ │ └── RegressionTest.java
│ │ │ │ ├── condition
│ │ │ │ │ └── OpenDALExceptionCondition.java
│ │ │ │ ├── LayerTest.java
│ │ │ │ ├── MetadataTest.java
│ │ │ │ ├── OperatorDuplicateTest.java
│ │ │ │ ├── OperatorInfoTest.java
│ │ │ │ ├── OperatorInputOutputStreamTest.java
│ │ │ │ ├── OperatorUtf8DecodeTest.java
│ │ │ │ └── UtilityTest.java
│ │ │ └── utility.rs
│ │ ├── tools
│ │ │ └── build.py
│ │ ├── upgrade.md
│ │ └── users.md
│ ├── nodejs
│ │ ├── .cargo
│ │ │ └── config.toml
│ │ ├── .gitignore
│ │ ├── .node-version
│ │ ├── .npmignore
│ │ ├── .npmrc
│ │ ├── .prettierignore
│ │ ├── benchmark
│ │ │ ├── deno.ts
│ │ │ ├── node.js
│ │ │ └── README.md
│ │ ├── build.rs
│ │ ├── Cargo.toml
│ │ ├── CONTRIBUTING.md
│ │ ├── DEPENDENCIES.md
│ │ ├── DEPENDENCIES.rust.tsv
│ │ ├── devbox.json
│ │ ├── devbox.lock
│ │ ├── generated.d.ts
│ │ ├── generated.js
│ │ ├── index.cjs
│ │ ├── index.d.ts
│ │ ├── index.mjs
│ │ ├── npm
│ │ │ ├── darwin-arm64
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ ├── darwin-x64
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ ├── linux-arm64-gnu
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ ├── linux-arm64-musl
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ ├── linux-x64-gnu
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ ├── linux-x64-musl
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ ├── win32-arm64-msvc
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ └── win32-x64-msvc
│ │ │ ├── package.json
│ │ │ └── README.md
│ │ ├── package.json
│ │ ├── pnpm-lock.yaml
│ │ ├── README.md
│ │ ├── scripts
│ │ │ └── header.mjs
│ │ ├── src
│ │ │ ├── capability.rs
│ │ │ ├── layer.rs
│ │ │ ├── lib.rs
│ │ │ └── options.rs
│ │ ├── tests
│ │ │ ├── service.test.mjs
│ │ │ ├── suites
│ │ │ │ ├── async.suite.mjs
│ │ │ │ ├── asyncDeleteOptions.suite.mjs
│ │ │ │ ├── asyncLister.suite.mjs
│ │ │ │ ├── asyncListOptions.suite.mjs
│ │ │ │ ├── asyncReadOptions.suite.mjs
│ │ │ │ ├── asyncStatOptions.suite.mjs
│ │ │ │ ├── asyncWriteOptions.suite.mjs
│ │ │ │ ├── index.mjs
│ │ │ │ ├── layer.suite.mjs
│ │ │ │ ├── services.suite.mjs
│ │ │ │ ├── sync.suite.mjs
│ │ │ │ ├── syncDeleteOptions.suite.mjs
│ │ │ │ ├── syncLister.suite.mjs
│ │ │ │ ├── syncListOptions.suite.mjs
│ │ │ │ ├── syncReadOptions.suite.mjs
│ │ │ │ ├── syncStatOptions.suite.mjs
│ │ │ │ └── syncWriteOptions.suite.mjs
│ │ │ └── utils.mjs
│ │ ├── theme
│ │ │ ├── index.tsx
│ │ │ └── package.json
│ │ ├── tsconfig.json
│ │ ├── tsconfig.theme.json
│ │ ├── typedoc.json
│ │ ├── upgrade.md
│ │ └── vitest.config.mjs
│ ├── python
│ │ ├── .gitignore
│ │ ├── benchmark
│ │ │ ├── async_opendal_benchmark.py
│ │ │ ├── async_origin_s3_benchmark_with_gevent.py
│ │ │ └── README.md
│ │ ├── Cargo.toml
│ │ ├── CONTRIBUTING.md
│ │ ├── DEPENDENCIES.md
│ │ ├── DEPENDENCIES.rust.tsv
│ │ ├── docs
│ │ │ ├── api
│ │ │ │ ├── async_file.md
│ │ │ │ ├── async_operator.md
│ │ │ │ ├── capability.md
│ │ │ │ ├── exceptions.md
│ │ │ │ ├── file.md
│ │ │ │ ├── layers.md
│ │ │ │ ├── operator.md
│ │ │ │ └── types.md
│ │ │ └── index.md
│ │ ├── justfile
│ │ ├── mkdocs.yml
│ │ ├── pyproject.toml
│ │ ├── pyrightconfig.json
│ │ ├── python
│ │ │ └── opendal
│ │ │ ├── __init__.py
│ │ │ ├── capability.pyi
│ │ │ ├── exceptions.pyi
│ │ │ ├── file.pyi
│ │ │ ├── layers.pyi
│ │ │ ├── operator.pyi
│ │ │ ├── py.typed
│ │ │ ├── services.pyi
│ │ │ └── types.pyi
│ │ ├── README.md
│ │ ├── ruff.toml
│ │ ├── src
│ │ │ ├── capability.rs
│ │ │ ├── errors.rs
│ │ │ ├── file.rs
│ │ │ ├── layers.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── metadata.rs
│ │ │ ├── operator.rs
│ │ │ ├── options.rs
│ │ │ ├── services.rs
│ │ │ └── utils.rs
│ │ ├── template
│ │ │ └── module.html.jinja2
│ │ ├── tests
│ │ │ ├── conftest.py
│ │ │ ├── test_async_check.py
│ │ │ ├── test_async_copy.py
│ │ │ ├── test_async_delete.py
│ │ │ ├── test_async_exists.py
│ │ │ ├── test_async_list.py
│ │ │ ├── test_async_pickle_types.py
│ │ │ ├── test_async_rename.py
│ │ │ ├── test_capability.py
│ │ │ ├── test_exceptions.py
│ │ │ ├── test_pickle_rw.py
│ │ │ ├── test_read.py
│ │ │ ├── test_sync_check.py
│ │ │ ├── test_sync_copy.py
│ │ │ ├── test_sync_delete.py
│ │ │ ├── test_sync_exists.py
│ │ │ ├── test_sync_list.py
│ │ │ ├── test_sync_pickle_types.py
│ │ │ ├── test_sync_rename.py
│ │ │ └── test_write.py
│ │ ├── upgrade.md
│ │ ├── users.md
│ │ └── uv.lock
│ └── README.md
├── CHANGELOG.md
├── CITATION.cff
├── CLAUDE.md
├── CONTRIBUTING.md
├── core
│ ├── benches
│ │ ├── ops
│ │ │ ├── main.rs
│ │ │ ├── read.rs
│ │ │ ├── README.md
│ │ │ ├── utils.rs
│ │ │ └── write.rs
│ │ ├── README.md
│ │ ├── types
│ │ │ ├── buffer.rs
│ │ │ ├── main.rs
│ │ │ ├── README.md
│ │ │ └── tasks.rs
│ │ ├── vs_fs
│ │ │ ├── Cargo.toml
│ │ │ ├── README.md
│ │ │ └── src
│ │ │ └── main.rs
│ │ └── vs_s3
│ │ ├── Cargo.toml
│ │ ├── README.md
│ │ └── src
│ │ └── main.rs
│ ├── Cargo.lock
│ ├── Cargo.toml
│ ├── CHANGELOG.md
│ ├── CONTRIBUTING.md
│ ├── core
│ │ ├── Cargo.toml
│ │ └── src
│ │ ├── blocking
│ │ │ ├── delete.rs
│ │ │ ├── list.rs
│ │ │ ├── mod.rs
│ │ │ ├── operator.rs
│ │ │ ├── read
│ │ │ │ ├── buffer_iterator.rs
│ │ │ │ ├── mod.rs
│ │ │ │ ├── reader.rs
│ │ │ │ ├── std_bytes_iterator.rs
│ │ │ │ └── std_reader.rs
│ │ │ └── write
│ │ │ ├── mod.rs
│ │ │ ├── std_writer.rs
│ │ │ └── writer.rs
│ │ ├── docs
│ │ │ ├── comparisons
│ │ │ │ ├── mod.rs
│ │ │ │ └── vs_object_store.md
│ │ │ ├── concepts.rs
│ │ │ ├── internals
│ │ │ │ ├── accessor.rs
│ │ │ │ ├── layer.rs
│ │ │ │ └── mod.rs
│ │ │ ├── mod.rs
│ │ │ ├── performance
│ │ │ │ ├── concurrent_write.md
│ │ │ │ ├── http_optimization.md
│ │ │ │ └── mod.rs
│ │ │ ├── rfcs
│ │ │ │ ├── 0000_example.md
│ │ │ │ ├── 0041_object_native_api.md
│ │ │ │ ├── 0044_error_handle.md
│ │ │ │ ├── 0057_auto_region.md
│ │ │ │ ├── 0069_object_stream.md
│ │ │ │ ├── 0090_limited_reader.md
│ │ │ │ ├── 0112_path_normalization.md
│ │ │ │ ├── 0191_async_streaming_io.md
│ │ │ │ ├── 0203_remove_credential.md
│ │ │ │ ├── 0221_create_dir.md
│ │ │ │ ├── 0247_retryable_error.md
│ │ │ │ ├── 0293_object_id.md
│ │ │ │ ├── 0337_dir_entry.md
│ │ │ │ ├── 0409_accessor_capabilities.md
│ │ │ │ ├── 0413_presign.md
│ │ │ │ ├── 0423_command_line_interface.md
│ │ │ │ ├── 0429_init_from_iter.md
│ │ │ │ ├── 0438_multipart.md
│ │ │ │ ├── 0443_gateway.md
│ │ │ │ ├── 0501_new_builder.md
│ │ │ │ ├── 0554_write_refactor.md
│ │ │ │ ├── 0561_list_metadata_reuse.md
│ │ │ │ ├── 0599_blocking_api.md
│ │ │ │ ├── 0623_redis_service.md
│ │ │ │ ├── 0627_split_capabilities.md
│ │ │ │ ├── 0661_path_in_accessor.md
│ │ │ │ ├── 0793_generic_kv_services.md
│ │ │ │ ├── 0926_object_reader.md
│ │ │ │ ├── 0977_refactor_error.md
│ │ │ │ ├── 1085_object_handler.md
│ │ │ │ ├── 1391_object_metadataer.md
│ │ │ │ ├── 1398_query_based_metadata.md
│ │ │ │ ├── 1420_object_writer.md
│ │ │ │ ├── 1477_remove_object_concept.md
│ │ │ │ ├── 1735_operation_extension.md
│ │ │ │ ├── 2083_writer_sink_api.md
│ │ │ │ ├── 2133_append_api.md
│ │ │ │ ├── 2299_chain_based_operator_api.md
│ │ │ │ ├── 2602_object_versioning.md
│ │ │ │ ├── 2758_merge_append_into_write.md
│ │ │ │ ├── 2774_lister_api.md
│ │ │ │ ├── 2779_list_with_metakey.md
│ │ │ │ ├── 2852_native_capability.md
│ │ │ │ ├── 2884_merge_range_read_into_read.md
│ │ │ │ ├── 3017_remove_write_copy_from.md
│ │ │ │ ├── 3197_config.md
│ │ │ │ ├── 3232_align_list_api.md
│ │ │ │ ├── 3243_list_prefix.md
│ │ │ │ ├── 3356_lazy_reader.md
│ │ │ │ ├── 3526_list_recursive.md
│ │ │ │ ├── 3574_concurrent_stat_in_list.md
│ │ │ │ ├── 3734_buffered_reader.md
│ │ │ │ ├── 3898_concurrent_writer.md
│ │ │ │ ├── 3911_deleter_api.md
│ │ │ │ ├── 4382_range_based_read.md
│ │ │ │ ├── 4638_executor.md
│ │ │ │ ├── 5314_remove_metakey.md
│ │ │ │ ├── 5444_operator_from_uri.md
│ │ │ │ ├── 5479_context.md
│ │ │ │ ├── 5485_conditional_reader.md
│ │ │ │ ├── 5495_list_with_deleted.md
│ │ │ │ ├── 5556_write_returns_metadata.md
│ │ │ │ ├── 5871_read_returns_metadata.md
│ │ │ │ ├── 6189_remove_native_blocking.md
│ │ │ │ ├── 6209_glob_support.md
│ │ │ │ ├── 6213_options_api.md
│ │ │ │ ├── 6370_foyer_integration.md
│ │ │ │ ├── 6678_simulate_layer.md
│ │ │ │ ├── 6707_capability_override_layer.md
│ │ │ │ ├── 6817_checksum.md
│ │ │ │ ├── 6828_core.md
│ │ │ │ ├── 7130_route_layer.md
│ │ │ │ ├── mod.rs
│ │ │ │ └── README.md
│ │ │ └── upgrade.md
│ │ ├── layers
│ │ │ ├── complete.rs
│ │ │ ├── correctness_check.rs
│ │ │ ├── error_context.rs
│ │ │ ├── http_client.rs
│ │ │ ├── mod.rs
│ │ │ ├── simulate.rs
│ │ │ └── type_eraser.rs
│ │ ├── lib.rs
│ │ ├── raw
│ │ │ ├── accessor.rs
│ │ │ ├── atomic_util.rs
│ │ │ ├── enum_utils.rs
│ │ │ ├── futures_util.rs
│ │ │ ├── http_util
│ │ │ │ ├── body.rs
│ │ │ │ ├── bytes_content_range.rs
│ │ │ │ ├── bytes_range.rs
│ │ │ │ ├── client.rs
│ │ │ │ ├── error.rs
│ │ │ │ ├── header.rs
│ │ │ │ ├── mod.rs
│ │ │ │ ├── multipart.rs
│ │ │ │ └── uri.rs
│ │ │ ├── layer.rs
│ │ │ ├── mod.rs
│ │ │ ├── oio
│ │ │ │ ├── buf
│ │ │ │ │ ├── flex_buf.rs
│ │ │ │ │ ├── mod.rs
│ │ │ │ │ ├── pooled_buf.rs
│ │ │ │ │ └── queue_buf.rs
│ │ │ │ ├── delete
│ │ │ │ │ ├── api.rs
│ │ │ │ │ ├── batch_delete.rs
│ │ │ │ │ ├── mod.rs
│ │ │ │ │ └── one_shot_delete.rs
│ │ │ │ ├── entry.rs
│ │ │ │ ├── list
│ │ │ │ │ ├── api.rs
│ │ │ │ │ ├── flat_list.rs
│ │ │ │ │ ├── hierarchy_list.rs
│ │ │ │ │ ├── mod.rs
│ │ │ │ │ ├── page_list.rs
│ │ │ │ │ └── prefix_list.rs
│ │ │ │ ├── mod.rs
│ │ │ │ ├── read
│ │ │ │ │ ├── api.rs
│ │ │ │ │ └── mod.rs
│ │ │ │ └── write
│ │ │ │ ├── api.rs
│ │ │ │ ├── append_write.rs
│ │ │ │ ├── block_write.rs
│ │ │ │ ├── mod.rs
│ │ │ │ ├── multipart_write.rs
│ │ │ │ ├── one_shot_write.rs
│ │ │ │ └── position_write.rs
│ │ │ ├── operation.rs
│ │ │ ├── ops.rs
│ │ │ ├── path_cache.rs
│ │ │ ├── path.rs
│ │ │ ├── rps.rs
│ │ │ ├── serde_util.rs
│ │ │ ├── std_io_util.rs
│ │ │ ├── time.rs
│ │ │ ├── tokio_util.rs
│ │ │ └── version.rs
│ │ ├── services
│ │ │ ├── memory
│ │ │ │ ├── backend.rs
│ │ │ │ ├── config.rs
│ │ │ │ ├── core.rs
│ │ │ │ ├── deleter.rs
│ │ │ │ ├── docs.md
│ │ │ │ ├── lister.rs
│ │ │ │ ├── mod.rs
│ │ │ │ └── writer.rs
│ │ │ └── mod.rs
│ │ └── types
│ │ ├── buffer.rs
│ │ ├── builder.rs
│ │ ├── capability.rs
│ │ ├── context
│ │ │ ├── mod.rs
│ │ │ ├── read.rs
│ │ │ └── write.rs
│ │ ├── delete
│ │ │ ├── deleter.rs
│ │ │ ├── futures_delete_sink.rs
│ │ │ ├── input.rs
│ │ │ └── mod.rs
│ │ ├── entry.rs
│ │ ├── error.rs
│ │ ├── execute
│ │ │ ├── api.rs
│ │ │ ├── executor.rs
│ │ │ ├── executors
│ │ │ │ ├── mod.rs
│ │ │ │ └── tokio_executor.rs
│ │ │ └── mod.rs
│ │ ├── list.rs
│ │ ├── metadata.rs
│ │ ├── mod.rs
│ │ ├── mode.rs
│ │ ├── operator
│ │ │ ├── builder.rs
│ │ │ ├── info.rs
│ │ │ ├── mod.rs
│ │ │ ├── operator_futures.rs
│ │ │ ├── operator.rs
│ │ │ ├── registry.rs
│ │ │ └── uri.rs
│ │ ├── options.rs
│ │ ├── read
│ │ │ ├── buffer_stream.rs
│ │ │ ├── futures_async_reader.rs
│ │ │ ├── futures_bytes_stream.rs
│ │ │ ├── mod.rs
│ │ │ └── reader.rs
│ │ └── write
│ │ ├── buffer_sink.rs
│ │ ├── futures_async_writer.rs
│ │ ├── futures_bytes_sink.rs
│ │ ├── mod.rs
│ │ └── writer.rs
│ ├── DEPENDENCIES.md
│ ├── DEPENDENCIES.rust.tsv
│ ├── edge
│ │ ├── file_write_on_full_disk
│ │ │ ├── Cargo.toml
│ │ │ ├── README.md
│ │ │ └── src
│ │ │ └── main.rs
│ │ ├── README.md
│ │ ├── s3_aws_assume_role_with_web_identity
│ │ │ ├── Cargo.toml
│ │ │ ├── README.md
│ │ │ └── src
│ │ │ └── main.rs
│ │ └── s3_read_on_wasm
│ │ ├── .gitignore
│ │ ├── Cargo.toml
│ │ ├── README.md
│ │ ├── src
│ │ │ └── lib.rs
│ │ └── webdriver.json
│ ├── fuzz
│ │ ├── .gitignore
│ │ ├── Cargo.toml
│ │ ├── fuzz_reader.rs
│ │ ├── fuzz_writer.rs
│ │ └── README.md
│ ├── layers
│ │ ├── async-backtrace
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── await-tree
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── capability-check
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── chaos
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── concurrent-limit
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── dtrace
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── fastmetrics
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── fastrace
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── foyer
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── hotpath
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── immutable-index
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── logging
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── metrics
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── mime-guess
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── observe-metrics-common
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── otelmetrics
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── oteltrace
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── prometheus
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── prometheus-client
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── retry
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── route
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── tail-cut
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── throttle
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── timeout
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ └── tracing
│ │ ├── Cargo.toml
│ │ └── src
│ │ └── lib.rs
│ ├── LICENSE
│ ├── README.md
│ ├── services
│ │ ├── aliyun-drive
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── alluxio
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── azblob
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── azdls
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── azfile
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── azure-common
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── b2
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── cacache
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── cloudflare-kv
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── model.rs
│ │ │ └── writer.rs
│ │ ├── compfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── reader.rs
│ │ │ └── writer.rs
│ │ ├── cos
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── d1
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── model.rs
│ │ │ └── writer.rs
│ │ ├── dashmap
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── dbfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── dropbox
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── builder.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── etcd
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── foundationdb
│ │ │ ├── build.rs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── fs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── reader.rs
│ │ │ └── writer.rs
│ │ ├── ftp
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── err.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── reader.rs
│ │ │ └── writer.rs
│ │ ├── gcs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── uri.rs
│ │ │ └── writer.rs
│ │ ├── gdrive
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── builder.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── ghac
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── github
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── mod.rs
│ │ │ └── writer.rs
│ │ ├── gridfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── hdfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── reader.rs
│ │ │ └── writer.rs
│ │ ├── hdfs-native
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── reader.rs
│ │ │ └── writer.rs
│ │ ├── http
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ └── lib.rs
│ │ ├── huggingface
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ └── lister.rs
│ │ ├── ipfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── ipld.rs
│ │ │ └── lib.rs
│ │ ├── ipmfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── builder.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── koofr
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── lakefs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── memcached
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── binary.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── mini_moka
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── moka
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── mongodb
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── monoiofs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── reader.rs
│ │ │ └── writer.rs
│ │ ├── mysql
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── obs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── onedrive
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── builder.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── graph_model.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── opfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ └── utils.rs
│ │ ├── oss
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── pcloud
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── persy
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── postgresql
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── redb
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── redis
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── delete.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── rocksdb
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── s3
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── compatible_services.md
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── mod.rs
│ │ │ └── writer.rs
│ │ ├── seafile
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── sftp
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── reader.rs
│ │ │ ├── utils.rs
│ │ │ └── writer.rs
│ │ ├── sled
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── sqlite
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── surrealdb
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── swift
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── compatible_services.md
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── tikv
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── upyun
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── vercel-artifacts
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── builder.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── vercel-blob
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── webdav
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── webhdfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── message.rs
│ │ │ └── writer.rs
│ │ └── yandex-disk
│ │ ├── Cargo.toml
│ │ └── src
│ │ ├── backend.rs
│ │ ├── config.rs
│ │ ├── core.rs
│ │ ├── deleter.rs
│ │ ├── docs.md
│ │ ├── error.rs
│ │ ├── lib.rs
│ │ ├── lister.rs
│ │ └── writer.rs
│ ├── src
│ │ └── lib.rs
│ ├── testkit
│ │ ├── Cargo.toml
│ │ └── src
│ │ ├── lib.rs
│ │ ├── read.rs
│ │ ├── utils.rs
│ │ └── write.rs
│ ├── tests
│ │ ├── behavior
│ │ │ ├── async_copy.rs
│ │ │ ├── async_create_dir.rs
│ │ │ ├── async_delete.rs
│ │ │ ├── async_list.rs
│ │ │ ├── async_presign.rs
│ │ │ ├── async_read.rs
│ │ │ ├── async_rename.rs
│ │ │ ├── async_stat.rs
│ │ │ ├── async_write.rs
│ │ │ ├── main.rs
│ │ │ ├── README.md
│ │ │ └── utils.rs
│ │ └── data
│ │ ├── normal_dir
│ │ │ └── .gitkeep
│ │ ├── normal_file.txt
│ │ ├── special_dir !@#$%^&()_+-=;',
│ │ │ └── .gitkeep
│ │ └── special_file !@#$%^&()_+-=;',.txt
│ ├── upgrade.md
│ └── users.md
├── deny.toml
├── DEPENDENCIES.md
├── dev
│ ├── Cargo.lock
│ ├── Cargo.toml
│ ├── README.md
│ └── src
│ ├── generate
│ │ ├── java.j2
│ │ ├── java.rs
│ │ ├── mod.rs
│ │ ├── parser.rs
│ │ ├── python.j2
│ │ └── python.rs
│ ├── main.rs
│ └── release
│ ├── mod.rs
│ └── package.rs
├── doap.rdf
├── fixtures
│ ├── alluxio
│ │ └── docker-compose-alluxio.yml
│ ├── azblob
│ │ └── docker-compose-azurite.yml
│ ├── data
│ │ ├── normal_dir
│ │ │ └── .gitkeep
│ │ ├── normal_file.txt
│ │ ├── special_dir !@#$%^&()_+-=;',
│ │ │ └── .gitkeep
│ │ └── special_file !@#$%^&()_+-=;',.txt
│ ├── etcd
│ │ ├── ca-key.pem
│ │ ├── ca.pem
│ │ ├── client-key.pem
│ │ ├── client.pem
│ │ ├── docker-compose-cluster.yml
│ │ ├── docker-compose-standalone-tls.yml
│ │ ├── docker-compose-standalone.yml
│ │ ├── server-key.pem
│ │ └── server.pem
│ ├── ftp
│ │ └── docker-compose-vsftpd.yml
│ ├── hdfs
│ │ ├── azurite-azblob-core-site.xml
│ │ ├── docker-compose-hdfs-cluster.yml
│ │ ├── gcs-core-site.xml
│ │ ├── hdfs-site.xml
│ │ └── minio-s3-core-site.xml
│ ├── http
│ │ ├── Caddyfile
│ │ ├── docker-compose-caddy.yml
│ │ ├── docker-compose-nginx.yml
│ │ └── nginx.conf
│ ├── libsql
│ │ ├── docker-compose-auth.yml
│ │ └── docker-compose.yml
│ ├── memcached
│ │ ├── docker-compose-memcached-with-auth.yml
│ │ └── docker-compose-memcached.yml
│ ├── mongodb
│ │ ├── docker-compose-basic-auth.yml
│ │ └── docker-compose-no-auth.yml
│ ├── mysql
│ │ ├── docker-compose.yml
│ │ └── init.sql
│ ├── postgresql
│ │ ├── docker-compose.yml
│ │ └── init.sql
│ ├── redis
│ │ ├── docker-compose-dragonfly.yml
│ │ ├── docker-compose-kvrocks.yml
│ │ ├── docker-compose-redis-cluster-tls.yml
│ │ ├── docker-compose-redis-cluster.yml
│ │ ├── docker-compose-redis-tls.yml
│ │ ├── docker-compose-redis.yml
│ │ └── ssl
│ │ ├── .gitignore
│ │ ├── ca.crt
│ │ ├── ca.key
│ │ ├── ca.srl
│ │ ├── README.md
│ │ ├── redis.crt
│ │ ├── redis.key
│ │ └── req.conf
│ ├── s3
│ │ ├── docker-compose-ceph-rados.yml
│ │ └── docker-compose-minio.yml
│ ├── seafile
│ │ └── docker-compose-seafile.yml
│ ├── sftp
│ │ ├── change_root_dir.sh
│ │ ├── docker-compose-sftp-with-default-root.yml
│ │ ├── docker-compose-sftp.yml
│ │ ├── health-check.sh
│ │ ├── test_ssh_key
│ │ └── test_ssh_key.pub
│ ├── sqlite
│ │ └── data.sql
│ ├── swift
│ │ ├── docker-compose-ceph-rados.yml
│ │ └── docker-compose-swift.yml
│ ├── tikv
│ │ ├── gen_cert.sh
│ │ ├── pd-tls.toml
│ │ ├── pd.toml
│ │ ├── ssl
│ │ │ ├── ca-key.pem
│ │ │ ├── ca.pem
│ │ │ ├── client-key.pem
│ │ │ ├── client.pem
│ │ │ ├── pd-server-key.pem
│ │ │ ├── pd-server.pem
│ │ │ ├── tikv-server-key.pem
│ │ │ └── tikv-server.pem
│ │ ├── tikv-tls.toml
│ │ └── tikv.toml
│ ├── webdav
│ │ ├── config
│ │ │ └── nginx
│ │ │ └── http.conf
│ │ ├── docker-compose-webdav-jfrog.yml
│ │ ├── docker-compose-webdav-nextcloud.yml
│ │ ├── docker-compose-webdav-owncloud.yml
│ │ ├── docker-compose-webdav-with-auth.yml
│ │ ├── docker-compose-webdav-with-empty-passwd.yml
│ │ ├── docker-compose-webdav.yml
│ │ └── health-check-nextcloud.sh
│ └── webhdfs
│ └── docker-compose-webhdfs.yml
├── justfile
├── LICENSE
├── licenserc.toml
├── NOTICE
├── README.md
├── rust-toolchain.toml
├── rustfmt.toml
└── scripts
├── constants.py
├── dependencies.py
├── merge_local_staging.py
├── README.md
├── verify.py
└── workspace.py
```
# Files
--------------------------------------------------------------------------------
/core/tests/behavior/async_list.rs:
--------------------------------------------------------------------------------
```rust
1 | // Licensed to the Apache Software Foundation (ASF) under one
2 | // or more contributor license agreements. See the NOTICE file
3 | // distributed with this work for additional information
4 | // regarding copyright ownership. The ASF licenses this file
5 | // to you under the Apache License, Version 2.0 (the
6 | // "License"); you may not use this file except in compliance
7 | // with the License. You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing,
12 | // software distributed under the License is distributed on an
13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 | // KIND, either express or implied. See the License for the
15 | // specific language governing permissions and limitations
16 | // under the License.
17 |
18 | use std::collections::HashMap;
19 | use std::collections::HashSet;
20 |
21 | use anyhow::Result;
22 | use futures::StreamExt;
23 | use futures::TryStreamExt;
24 | use futures::stream::FuturesUnordered;
25 | use log::debug;
26 |
27 | use crate::*;
28 |
29 | pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
30 | let cap = op.info().full_capability();
31 |
32 | if cap.read && cap.write && cap.list {
33 | tests.extend(async_trials!(
34 | op,
35 | test_check,
36 | test_list_dir,
37 | test_list_prefix,
38 | test_list_rich_dir,
39 | test_list_empty_dir,
40 | test_list_non_exist_dir,
41 | test_list_sub_dir,
42 | test_list_nested_dir,
43 | test_list_dir_with_file_path,
44 | test_list_with_start_after,
45 | test_list_non_exist_dir_with_recursive,
46 | test_list_dir_with_recursive,
47 | test_list_dir_with_recursive_no_trailing_slash,
48 | test_list_file_with_recursive,
49 | test_list_root_with_recursive,
50 | test_remove_all,
51 | test_list_files_with_versions,
52 | test_list_with_versions_and_limit,
53 | test_list_with_versions_and_start_after,
54 | test_list_files_with_deleted
55 | ))
56 | }
57 |
58 | if cap.read && !cap.write && cap.list {
59 | tests.extend(async_trials!(op, test_list_only))
60 | }
61 | }
62 |
63 | /// Check should be OK.
64 | pub async fn test_check(op: Operator) -> Result<()> {
65 | op.check().await.expect("operator check is ok");
66 |
67 | Ok(())
68 | }
69 |
70 | /// List dir should return newly created file.
71 | pub async fn test_list_dir(op: Operator) -> Result<()> {
72 | let parent = uuid::Uuid::new_v4().to_string();
73 | let path = format!("{parent}/{}", uuid::Uuid::new_v4());
74 | debug!("Generate a random file: {}", &path);
75 | let (content, size) = gen_bytes(op.info().full_capability());
76 |
77 | op.write(&path, content).await.expect("write must succeed");
78 |
79 | let mut obs = op.lister(&format!("{parent}/")).await?;
80 | let mut found = false;
81 | while let Some(de) = obs.try_next().await? {
82 | let meta = op.stat(de.path()).await?;
83 | if de.path() == path {
84 | assert_eq!(meta.mode(), EntryMode::FILE);
85 |
86 | assert_eq!(meta.content_length(), size as u64);
87 |
88 | found = true
89 | }
90 | }
91 | assert!(found, "file should be found in list");
92 |
93 | op.delete(&path).await.expect("delete must succeed");
94 | Ok(())
95 | }
96 |
97 | /// List prefix should return newly created file.
98 | pub async fn test_list_prefix(op: Operator) -> Result<()> {
99 | let path = uuid::Uuid::new_v4().to_string();
100 | debug!("Generate a random file: {}", &path);
101 | let (content, _) = gen_bytes(op.info().full_capability());
102 |
103 | op.write(&path, content).await.expect("write must succeed");
104 |
105 | let obs = op.list(&path).await?;
106 | assert_eq!(obs.len(), 1);
107 | assert_eq!(obs[0].path(), path);
108 | assert_eq!(obs[0].metadata().mode(), EntryMode::FILE);
109 |
110 | op.delete(&path).await.expect("delete must succeed");
111 | Ok(())
112 | }
113 |
114 | /// listing a directory, which contains more objects than a single page can take.
115 | pub async fn test_list_rich_dir(op: Operator) -> Result<()> {
116 | // Gdrive think that this test is an abuse of their service and redirect us
117 | // to an infinite loop. Let's ignore this test for gdrive.
118 | #[cfg(feature = "services-gdrive")]
119 | if op.info().scheme() == services::GDRIVE_SCHEME {
120 | return Ok(());
121 | }
122 |
123 | let parent = "test_list_rich_dir/";
124 | op.create_dir(parent).await?;
125 |
126 | let mut expected: Vec<String> = (0..=10).map(|num| format!("{parent}file-{num}")).collect();
127 | for path in expected.iter() {
128 | op.write(path, "test_list_rich_dir").await?;
129 | }
130 | expected.push(parent.to_string());
131 |
132 | let mut objects = op.lister_with(parent).limit(5).await?;
133 | let mut actual = vec![];
134 | while let Some(o) = objects.try_next().await? {
135 | let path = o.path().to_string();
136 | actual.push(path)
137 | }
138 | expected.sort_unstable();
139 | actual.sort_unstable();
140 |
141 | assert_eq!(actual, expected);
142 |
143 | op.delete_with(parent).recursive(true).await?;
144 | Ok(())
145 | }
146 |
147 | /// List empty dir should return itself.
148 | pub async fn test_list_empty_dir(op: Operator) -> Result<()> {
149 | let dir = format!("{}/", uuid::Uuid::new_v4());
150 |
151 | op.create_dir(&dir).await.expect("write must succeed");
152 |
153 | // List "dir/" should return "dir/".
154 | let mut obs = op.lister(&dir).await?;
155 | let mut objects = HashMap::new();
156 | while let Some(de) = obs.try_next().await? {
157 | objects.insert(de.path().to_string(), de);
158 | }
159 | assert_eq!(
160 | objects.len(),
161 | 1,
162 | "only return the dir itself, but found: {objects:?}"
163 | );
164 | assert_eq!(
165 | objects[&dir].metadata().mode(),
166 | EntryMode::DIR,
167 | "given dir should exist and must be dir, but found: {objects:?}"
168 | );
169 |
170 | // List "dir" should return "dir/".
171 | let mut obs = op.lister(dir.trim_end_matches('/')).await?;
172 | let mut objects = HashMap::new();
173 | while let Some(de) = obs.try_next().await? {
174 | objects.insert(de.path().to_string(), de);
175 | }
176 | assert_eq!(
177 | objects.len(),
178 | 1,
179 | "only return the dir itself, but found: {objects:?}"
180 | );
181 | assert_eq!(
182 | objects[&dir].metadata().mode(),
183 | EntryMode::DIR,
184 | "given dir should exist and must be dir, but found: {objects:?}"
185 | );
186 |
187 | // List "dir/" recursively should return "dir/".
188 | let mut obs = op.lister_with(&dir).recursive(true).await?;
189 | let mut objects = HashMap::new();
190 | while let Some(de) = obs.try_next().await? {
191 | objects.insert(de.path().to_string(), de);
192 | }
193 | assert_eq!(
194 | objects.len(),
195 | 1,
196 | "only return the dir itself, but found: {objects:?}"
197 | );
198 | assert_eq!(
199 | objects[&dir].metadata().mode(),
200 | EntryMode::DIR,
201 | "given dir should exist and must be dir, but found: {objects:?}"
202 | );
203 |
204 | // List "dir" recursively should return "dir/".
205 | let mut obs = op
206 | .lister_with(dir.trim_end_matches('/'))
207 | .recursive(true)
208 | .await?;
209 | let mut objects = HashMap::new();
210 | while let Some(de) = obs.try_next().await? {
211 | objects.insert(de.path().to_string(), de);
212 | }
213 | assert_eq!(objects.len(), 1, "only return the dir itself");
214 | assert_eq!(
215 | objects[&dir].metadata().mode(),
216 | EntryMode::DIR,
217 | "given dir should exist and must be dir"
218 | );
219 |
220 | op.delete(&dir).await.expect("delete must succeed");
221 | Ok(())
222 | }
223 |
224 | /// List non exist dir should return nothing.
225 | pub async fn test_list_non_exist_dir(op: Operator) -> Result<()> {
226 | let dir = format!("{}/", uuid::Uuid::new_v4());
227 |
228 | let mut obs = op.lister(&dir).await?;
229 | let mut objects = HashMap::new();
230 | while let Some(de) = obs.try_next().await? {
231 | objects.insert(de.path().to_string(), de);
232 | }
233 | debug!("got objects: {objects:?}");
234 |
235 | assert_eq!(objects.len(), 0, "dir should only return empty");
236 | Ok(())
237 | }
238 |
239 | /// List dir should return correct sub dir.
240 | pub async fn test_list_sub_dir(op: Operator) -> Result<()> {
241 | let path = format!("{}/", uuid::Uuid::new_v4());
242 |
243 | op.create_dir(&path).await.expect("create must succeed");
244 |
245 | let mut obs = op.lister("/").await?;
246 | let mut found = false;
247 | let mut entries = vec![];
248 | while let Some(de) = obs.try_next().await? {
249 | if de.path() == path {
250 | let meta = op.stat(&path).await?;
251 | assert_eq!(meta.mode(), EntryMode::DIR);
252 | assert_eq!(de.name(), path);
253 |
254 | found = true
255 | }
256 | entries.push(de)
257 | }
258 | assert!(
259 | found,
260 | "dir should be found in list, but only got: {entries:?}"
261 | );
262 |
263 | op.delete(&path).await.expect("delete must succeed");
264 | Ok(())
265 | }
266 |
267 | /// List dir should also to list nested dir.
268 | pub async fn test_list_nested_dir(op: Operator) -> Result<()> {
269 | let parent = format!("{}/", uuid::Uuid::new_v4());
270 | op.create_dir(&parent)
271 | .await
272 | .expect("create dir must succeed");
273 |
274 | let dir = format!("{parent}{}/", uuid::Uuid::new_v4());
275 | op.create_dir(&dir).await.expect("create must succeed");
276 |
277 | let file_name = uuid::Uuid::new_v4().to_string();
278 | let file_path = format!("{dir}{file_name}");
279 | op.write(&file_path, "test_list_nested_dir")
280 | .await
281 | .expect("create must succeed");
282 |
283 | let dir_name = format!("{}/", uuid::Uuid::new_v4());
284 | let dir_path = format!("{dir}{dir_name}");
285 | op.create_dir(&dir_path).await.expect("create must succeed");
286 |
287 | let obs = op.list(&parent).await?;
288 | assert_eq!(obs.len(), 2, "parent should got 2 entry");
289 | let objects: HashMap<&str, &Entry> = obs.iter().map(|e| (e.path(), e)).collect();
290 | assert_eq!(
291 | objects
292 | .get(parent.as_str())
293 | .expect("parent should be found in list")
294 | .metadata()
295 | .mode(),
296 | EntryMode::DIR
297 | );
298 | assert_eq!(
299 | objects
300 | .get(dir.as_str())
301 | .expect("dir should be found in list")
302 | .metadata()
303 | .mode(),
304 | EntryMode::DIR
305 | );
306 |
307 | let mut obs = op.lister(&dir).await?;
308 | let mut objects = HashMap::new();
309 |
310 | while let Some(de) = obs.try_next().await? {
311 | objects.insert(de.path().to_string(), de);
312 | }
313 | debug!("got objects: {objects:?}");
314 |
315 | assert_eq!(objects.len(), 3, "dir should only got 3 objects");
316 |
317 | // Check file
318 | let meta = op
319 | .stat(
320 | objects
321 | .get(&file_path)
322 | .expect("file should be found in list")
323 | .path(),
324 | )
325 | .await?;
326 | assert_eq!(meta.mode(), EntryMode::FILE);
327 | assert_eq!(meta.content_length(), 20);
328 |
329 | // Check dir
330 | let meta = op
331 | .stat(
332 | objects
333 | .get(&dir_path)
334 | .expect("file should be found in list")
335 | .path(),
336 | )
337 | .await?;
338 | assert_eq!(meta.mode(), EntryMode::DIR);
339 |
340 | op.delete(&file_path).await.expect("delete must succeed");
341 | op.delete(&dir_path).await.expect("delete must succeed");
342 | op.delete(&dir).await.expect("delete must succeed");
343 | Ok(())
344 | }
345 |
346 | /// List with path file should auto add / suffix.
347 | pub async fn test_list_dir_with_file_path(op: Operator) -> Result<()> {
348 | let parent = uuid::Uuid::new_v4().to_string();
349 | let file = format!("{parent}/{}", uuid::Uuid::new_v4());
350 |
351 | let (content, _) = gen_bytes(op.info().full_capability());
352 | op.write(&file, content).await?;
353 |
354 | let obs = op.list(&parent).await?;
355 | assert_eq!(obs.len(), 1);
356 | assert_eq!(obs[0].path(), format!("{parent}/"));
357 | assert_eq!(obs[0].metadata().mode(), EntryMode::DIR);
358 |
359 | op.delete(&file).await?;
360 |
361 | Ok(())
362 | }
363 |
364 | /// List with start after should start listing after the specified key
365 | pub async fn test_list_with_start_after(op: Operator) -> Result<()> {
366 | if !op.info().full_capability().list_with_start_after {
367 | return Ok(());
368 | }
369 |
370 | let dir = &format!("{}/", uuid::Uuid::new_v4());
371 | op.create_dir(dir).await?;
372 |
373 | let given: Vec<String> = ["file-0", "file-1", "file-2", "file-3", "file-4", "file-5"]
374 | .iter()
375 | .map(|name| format!("{dir}{name}-{}", uuid::Uuid::new_v4()))
376 | .collect();
377 |
378 | given
379 | .iter()
380 | .map(|name| async {
381 | op.write(name, "content")
382 | .await
383 | .expect("create must succeed");
384 | })
385 | .collect::<FuturesUnordered<_>>()
386 | .collect::<Vec<_>>()
387 | .await;
388 |
389 | let mut objects = op.lister_with(dir).start_after(&given[2]).await?;
390 | let mut actual = vec![];
391 | while let Some(o) = objects.try_next().await? {
392 | if o.path() != dir {
393 | let path = o.path().to_string();
394 | actual.push(path)
395 | }
396 | }
397 |
398 | let expected: Vec<String> = given.into_iter().skip(3).collect();
399 |
400 | assert_eq!(expected, actual);
401 |
402 | op.delete_with(dir).recursive(true).await?;
403 |
404 | Ok(())
405 | }
406 |
407 | pub async fn test_list_non_exist_dir_with_recursive(op: Operator) -> Result<()> {
408 | let dir = format!("{}/", uuid::Uuid::new_v4());
409 |
410 | let mut obs = op.lister_with(&dir).recursive(true).await?;
411 | let mut objects = HashMap::new();
412 | while let Some(de) = obs.try_next().await? {
413 | objects.insert(de.path().to_string(), de);
414 | }
415 | debug!("got objects: {objects:?}");
416 |
417 | assert_eq!(objects.len(), 0, "dir should only return empty");
418 | Ok(())
419 | }
420 |
421 | pub async fn test_list_root_with_recursive(op: Operator) -> Result<()> {
422 | op.create_dir("/").await?;
423 |
424 | let w = op.lister_with("").recursive(true).await?;
425 | let actual = w
426 | .try_collect::<Vec<_>>()
427 | .await?
428 | .into_iter()
429 | .map(|v| v.path().to_string())
430 | .collect::<HashSet<_>>();
431 |
432 | assert!(actual.contains("/"), "empty root should return itself");
433 | assert!(!actual.contains(""), "empty root should not return empty");
434 | Ok(())
435 | }
436 |
437 | // Walk top down should output as expected
438 | pub async fn test_list_dir_with_recursive(op: Operator) -> Result<()> {
439 | let parent = uuid::Uuid::new_v4().to_string();
440 |
441 | let paths = [
442 | "x/", "x/x/", "x/x/x/", "x/x/x/x/", "x/x/x/y", "x/x/y", "x/y", "x/yy",
443 | ];
444 | for path in paths.iter() {
445 | if path.ends_with('/') {
446 | op.create_dir(&format!("{parent}/{path}")).await?;
447 | } else {
448 | op.write(&format!("{parent}/{path}"), "test_scan").await?;
449 | }
450 | }
451 | let w = op
452 | .lister_with(&format!("{parent}/x/"))
453 | .recursive(true)
454 | .await?;
455 | let mut actual = w
456 | .try_collect::<Vec<_>>()
457 | .await?
458 | .into_iter()
459 | .map(|v| {
460 | v.path()
461 | .strip_prefix(&format!("{parent}/"))
462 | .unwrap()
463 | .to_string()
464 | })
465 | .collect::<Vec<_>>();
466 | actual.sort();
467 |
468 | let expected = vec![
469 | "x/", "x/x/", "x/x/x/", "x/x/x/x/", "x/x/x/y", "x/x/y", "x/y", "x/yy",
470 | ];
471 | assert_eq!(actual, expected);
472 | Ok(())
473 | }
474 |
475 | // same as test_list_dir_with_recursive except listing 'x' instead of 'x/'
476 | pub async fn test_list_dir_with_recursive_no_trailing_slash(op: Operator) -> Result<()> {
477 | let parent = uuid::Uuid::new_v4().to_string();
478 |
479 | let paths = [
480 | "x/", "x/x/", "x/x/x/", "x/x/x/x/", "x/x/x/y", "x/x/y", "x/y", "x/yy",
481 | ];
482 | for path in paths.iter() {
483 | if path.ends_with('/') {
484 | op.create_dir(&format!("{parent}/{path}")).await?;
485 | } else {
486 | op.write(&format!("{parent}/{path}"), "test_scan").await?;
487 | }
488 | }
489 | let w = op
490 | .lister_with(&format!("{parent}/x"))
491 | .recursive(true)
492 | .await?;
493 | let mut actual = w
494 | .try_collect::<Vec<_>>()
495 | .await?
496 | .into_iter()
497 | .map(|v| {
498 | v.path()
499 | .strip_prefix(&format!("{parent}/"))
500 | .unwrap()
501 | .to_string()
502 | })
503 | .collect::<Vec<_>>();
504 | actual.sort();
505 |
506 | let expected = paths.to_vec();
507 | assert_eq!(actual, expected);
508 | Ok(())
509 | }
510 |
511 | pub async fn test_list_file_with_recursive(op: Operator) -> Result<()> {
512 | let parent = uuid::Uuid::new_v4().to_string();
513 |
514 | let paths = ["y", "yy"];
515 | for path in paths.iter() {
516 | if path.ends_with('/') {
517 | op.create_dir(&format!("{parent}/{path}")).await?;
518 | } else {
519 | op.write(&format!("{parent}/{path}"), "test_scan").await?;
520 | }
521 | }
522 | let w = op
523 | .lister_with(&format!("{parent}/y"))
524 | .recursive(true)
525 | .await?;
526 | let mut actual = w
527 | .try_collect::<Vec<_>>()
528 | .await?
529 | .into_iter()
530 | .map(|v| {
531 | v.path()
532 | .strip_prefix(&format!("{parent}/"))
533 | .unwrap()
534 | .to_string()
535 | })
536 | .collect::<Vec<_>>();
537 | actual.sort();
538 |
539 | let expected = vec!["y", "yy"];
540 | assert_eq!(actual, expected);
541 | Ok(())
542 | }
543 |
544 | // Remove all should remove all in this path.
545 | pub async fn test_remove_all(op: Operator) -> Result<()> {
546 | let parent = uuid::Uuid::new_v4().to_string();
547 |
548 | let expected = [
549 | "x/", "x/y", "x/x/", "x/x/y", "x/x/x/", "x/x/x/y", "x/x/x/x/",
550 | ];
551 | for path in expected.iter() {
552 | if path.ends_with('/') {
553 | op.create_dir(&format!("{parent}/{path}")).await?;
554 | } else {
555 | op.write(&format!("{parent}/{path}"), "test_scan").await?;
556 | }
557 | }
558 |
559 | op.delete_with(&format!("{parent}/x/"))
560 | .recursive(true)
561 | .await?;
562 |
563 | for path in expected.iter() {
564 | if path.ends_with('/') {
565 | continue;
566 | }
567 | assert!(
568 | !op.exists(&format!("{parent}/{path}")).await?,
569 | "{parent}/{path} should be removed"
570 | )
571 | }
572 | Ok(())
573 | }
574 |
575 | /// Stat normal file and dir should return metadata
576 | pub async fn test_list_only(op: Operator) -> Result<()> {
577 | let mut entries = HashMap::new();
578 |
579 | let mut ds = op.lister("/").await?;
580 | while let Some(de) = ds.try_next().await? {
581 | entries.insert(de.path().to_string(), op.stat(de.path()).await?.mode());
582 | }
583 |
584 | assert_eq!(entries["normal_file.txt"], EntryMode::FILE);
585 | assert_eq!(
586 | entries["special_file !@#$%^&()_+-=;',.txt"],
587 | EntryMode::FILE
588 | );
589 |
590 | assert_eq!(entries["normal_dir/"], EntryMode::DIR);
591 | assert_eq!(entries["special_dir !@#$%^&()_+-=;',/"], EntryMode::DIR);
592 |
593 | Ok(())
594 | }
595 |
596 | pub async fn test_list_files_with_versions(op: Operator) -> Result<()> {
597 | if !op.info().full_capability().list_with_versions {
598 | return Ok(());
599 | }
600 |
601 | let parent = TEST_FIXTURE.new_dir_path();
602 | let file_name = TEST_FIXTURE.new_file_path();
603 | let file_path = format!("{parent}{file_name}");
604 | op.write(file_path.as_str(), "1").await?;
605 | op.write(file_path.as_str(), "2").await?;
606 |
607 | let mut ds = op.list_with(parent.as_str()).versions(true).await?;
608 | ds.retain(|de| de.path() != parent.as_str());
609 |
610 | assert_eq!(ds.len(), 2);
611 | assert_ne!(ds[0].metadata().version(), ds[1].metadata().version());
612 | for de in ds {
613 | assert_eq!(de.name(), file_name);
614 | let meta = de.metadata();
615 | assert_eq!(meta.mode(), EntryMode::FILE);
616 | }
617 |
618 | Ok(())
619 | }
620 |
621 | pub async fn test_list_files_with_deleted(op: Operator) -> Result<()> {
622 | if !op.info().full_capability().list_with_deleted {
623 | return Ok(());
624 | }
625 |
626 | let parent = TEST_FIXTURE.new_dir_path();
627 | let file_name = TEST_FIXTURE.new_file_path();
628 | let file_path = format!("{parent}{file_name}");
629 | op.write(file_path.as_str(), "1").await?;
630 |
631 | // List with deleted should include self too.
632 | let ds = op.list_with(&file_path).deleted(true).await?;
633 | assert_eq!(
634 | ds.len(),
635 | 1,
636 | "list with deleted should contain current active file version"
637 | );
638 |
639 | op.write(file_path.as_str(), "2").await?;
640 | op.delete(file_path.as_str()).await?;
641 |
642 | // This file has been deleted, list with deleted should contain its versions and delete marker.
643 | let mut ds = op.list_with(&file_path).deleted(true).await?;
644 | ds.retain(|de| de.path() == file_path && de.metadata().is_deleted());
645 |
646 | assert_eq!(
647 | ds.len(),
648 | 1,
649 | "deleted file should be found and only have one"
650 | );
651 |
652 | Ok(())
653 | }
654 |
655 | // listing a directory with version, which contains more object versions than a page can take
656 | pub async fn test_list_with_versions_and_limit(op: Operator) -> Result<()> {
657 | // Gdrive think that this test is an abuse of their service and redirect us
658 | // to an infinite loop. Let's ignore this test for gdrive.
659 | #[cfg(feature = "services-gdrive")]
660 | if op.info().scheme() == services::GDRIVE_SCHEME {
661 | return Ok(());
662 | }
663 | if !op.info().full_capability().list_with_versions {
664 | return Ok(());
665 | }
666 |
667 | let parent = "test_list_with_version_and_limit/";
668 | op.create_dir(parent).await?;
669 |
670 | let expected: Vec<String> = (0..=10).map(|num| format!("{parent}file-{num}")).collect();
671 | for path in expected.iter() {
672 | // each file has 2 versions
673 | op.write(path, "1").await?;
674 | op.write(path, "2").await?;
675 | }
676 | let mut expected: Vec<String> = expected
677 | .into_iter()
678 | .flat_map(|v| std::iter::repeat_n(v, 2))
679 | .collect();
680 | expected.push(parent.to_string());
681 |
682 | let mut objects = op.lister_with(parent).versions(true).limit(5).await?;
683 | let mut actual = vec![];
684 | while let Some(o) = objects.try_next().await? {
685 | let path = o.path().to_string();
686 | actual.push(path)
687 | }
688 | expected.sort_unstable();
689 | actual.sort_unstable();
690 |
691 | assert_eq!(actual, expected);
692 |
693 | op.delete_with(parent).recursive(true).await?;
694 | Ok(())
695 | }
696 |
697 | pub async fn test_list_with_versions_and_start_after(op: Operator) -> Result<()> {
698 | if !op.info().full_capability().list_with_versions {
699 | return Ok(());
700 | }
701 |
702 | let dir = &format!("{}/", uuid::Uuid::new_v4());
703 |
704 | let given: Vec<String> = ["file-0", "file-1", "file-2", "file-3", "file-4", "file-5"]
705 | .iter()
706 | .map(|name| format!("{dir}{name}-{}", uuid::Uuid::new_v4()))
707 | .collect();
708 |
709 | given
710 | .iter()
711 | .map(|name| async {
712 | op.write(name, "1").await.expect("write must succeed");
713 | op.write(name, "2").await.expect("write must succeed");
714 | })
715 | .collect::<FuturesUnordered<_>>()
716 | .collect::<Vec<_>>()
717 | .await;
718 |
719 | let mut objects = op
720 | .lister_with(dir)
721 | .versions(true)
722 | .start_after(&given[2])
723 | .await?;
724 | let mut actual = vec![];
725 | while let Some(o) = objects.try_next().await? {
726 | let path = o.path().to_string();
727 | actual.push(path)
728 | }
729 |
730 | let expected: Vec<String> = given.into_iter().skip(3).collect();
731 | let mut expected: Vec<String> = expected
732 | .into_iter()
733 | .flat_map(|v| std::iter::repeat_n(v, 2))
734 | .collect();
735 |
736 | expected.sort_unstable();
737 | actual.sort_unstable();
738 | assert_eq!(expected, actual);
739 |
740 | op.delete_with(dir).recursive(true).await?;
741 |
742 | Ok(())
743 | }
744 |
```
--------------------------------------------------------------------------------
/core/services/cos/src/core.rs:
--------------------------------------------------------------------------------
```rust
1 | // Licensed to the Apache Software Foundation (ASF) under one
2 | // or more contributor license agreements. See the NOTICE file
3 | // distributed with this work for additional information
4 | // regarding copyright ownership. The ASF licenses this file
5 | // to you under the Apache License, Version 2.0 (the
6 | // "License"); you may not use this file except in compliance
7 | // with the License. You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing,
12 | // software distributed under the License is distributed on an
13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 | // KIND, either express or implied. See the License for the
15 | // specific language governing permissions and limitations
16 | // under the License.
17 |
18 | use std::fmt::Debug;
19 | use std::sync::Arc;
20 |
21 | use bytes::Bytes;
22 | use http::Request;
23 | use http::Response;
24 | use http::header::CACHE_CONTROL;
25 | use http::header::CONTENT_DISPOSITION;
26 | use http::header::CONTENT_LENGTH;
27 | use http::header::CONTENT_TYPE;
28 | use http::header::IF_MATCH;
29 | use http::header::IF_MODIFIED_SINCE;
30 | use http::header::IF_NONE_MATCH;
31 | use http::header::IF_UNMODIFIED_SINCE;
32 | use reqsign::TencentCosCredential;
33 | use reqsign::TencentCosCredentialLoader;
34 | use reqsign::TencentCosSigner;
35 | use serde::Deserialize;
36 | use serde::Serialize;
37 |
38 | use opendal_core::Buffer;
39 | use opendal_core::Error;
40 | use opendal_core::ErrorKind;
41 | use opendal_core::Result;
42 | use opendal_core::raw::*;
43 |
44 | pub mod constants {
45 | pub const COS_QUERY_VERSION_ID: &str = "versionId";
46 |
47 | pub const X_COS_VERSION_ID: &str = "x-cos-version-id";
48 | }
49 |
50 | pub struct CosCore {
51 | pub info: Arc<AccessorInfo>,
52 | pub bucket: String,
53 | pub root: String,
54 | pub endpoint: String,
55 |
56 | pub signer: TencentCosSigner,
57 | pub loader: TencentCosCredentialLoader,
58 | }
59 |
60 | impl Debug for CosCore {
61 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
62 | f.debug_struct("CosCore")
63 | .field("root", &self.root)
64 | .field("bucket", &self.bucket)
65 | .field("endpoint", &self.endpoint)
66 | .finish_non_exhaustive()
67 | }
68 | }
69 |
70 | impl CosCore {
71 | async fn load_credential(&self) -> Result<Option<TencentCosCredential>> {
72 | let cred = self
73 | .loader
74 | .load()
75 | .await
76 | .map_err(new_request_credential_error)?;
77 |
78 | if let Some(cred) = cred {
79 | return Ok(Some(cred));
80 | }
81 |
82 | Err(Error::new(
83 | ErrorKind::PermissionDenied,
84 | "no valid credential found and anonymous access is not allowed",
85 | ))
86 | }
87 |
88 | pub async fn sign<T>(&self, req: &mut Request<T>) -> Result<()> {
89 | let cred = if let Some(cred) = self.load_credential().await? {
90 | cred
91 | } else {
92 | return Ok(());
93 | };
94 |
95 | self.signer.sign(req, &cred).map_err(new_request_sign_error)
96 | }
97 |
98 | pub async fn sign_query<T>(&self, req: &mut Request<T>, duration: Duration) -> Result<()> {
99 | let cred = if let Some(cred) = self.load_credential().await? {
100 | cred
101 | } else {
102 | return Ok(());
103 | };
104 |
105 | self.signer
106 | .sign_query(req, duration, &cred)
107 | .map_err(new_request_sign_error)
108 | }
109 |
110 | #[inline]
111 | pub async fn send(&self, req: Request<Buffer>) -> Result<Response<Buffer>> {
112 | self.info.http_client().send(req).await
113 | }
114 | }
115 |
116 | impl CosCore {
117 | pub async fn cos_get_object(
118 | &self,
119 | path: &str,
120 | range: BytesRange,
121 | args: &OpRead,
122 | ) -> Result<Response<HttpBody>> {
123 | let mut req = self.cos_get_object_request(path, range, args)?;
124 |
125 | self.sign(&mut req).await?;
126 |
127 | self.info.http_client().fetch(req).await
128 | }
129 |
130 | pub fn cos_get_object_request(
131 | &self,
132 | path: &str,
133 | range: BytesRange,
134 | args: &OpRead,
135 | ) -> Result<Request<Buffer>> {
136 | let p = build_abs_path(&self.root, path);
137 |
138 | let mut url = format!("{}/{}", self.endpoint, percent_encode_path(&p));
139 |
140 | let mut query_args = Vec::new();
141 | if let Some(version) = args.version() {
142 | query_args.push(format!(
143 | "{}={}",
144 | constants::COS_QUERY_VERSION_ID,
145 | percent_decode_path(version)
146 | ))
147 | }
148 | if !query_args.is_empty() {
149 | url.push_str(&format!("?{}", query_args.join("&")));
150 | }
151 |
152 | let mut req = Request::get(&url);
153 |
154 | if let Some(if_match) = args.if_match() {
155 | req = req.header(IF_MATCH, if_match);
156 | }
157 |
158 | if !range.is_full() {
159 | req = req.header(http::header::RANGE, range.to_header())
160 | }
161 |
162 | if let Some(if_none_match) = args.if_none_match() {
163 | req = req.header(IF_NONE_MATCH, if_none_match);
164 | }
165 |
166 | if let Some(if_modified_since) = args.if_modified_since() {
167 | req = req.header(IF_MODIFIED_SINCE, if_modified_since.format_http_date());
168 | }
169 |
170 | if let Some(if_unmodified_since) = args.if_unmodified_since() {
171 | req = req.header(IF_UNMODIFIED_SINCE, if_unmodified_since.format_http_date());
172 | }
173 |
174 | let req = req.extension(Operation::Read);
175 |
176 | let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
177 |
178 | Ok(req)
179 | }
180 |
181 | pub fn cos_put_object_request(
182 | &self,
183 | path: &str,
184 | size: Option<u64>,
185 | args: &OpWrite,
186 | body: Buffer,
187 | ) -> Result<Request<Buffer>> {
188 | let p = build_abs_path(&self.root, path);
189 |
190 | let url = format!("{}/{}", self.endpoint, percent_encode_path(&p));
191 |
192 | let mut req = Request::put(&url);
193 |
194 | if let Some(size) = size {
195 | req = req.header(CONTENT_LENGTH, size)
196 | }
197 | if let Some(cache_control) = args.cache_control() {
198 | req = req.header(CACHE_CONTROL, cache_control)
199 | }
200 | if let Some(pos) = args.content_disposition() {
201 | req = req.header(CONTENT_DISPOSITION, pos)
202 | }
203 | if let Some(mime) = args.content_type() {
204 | req = req.header(CONTENT_TYPE, mime)
205 | }
206 |
207 | // For a bucket which has never enabled versioning, you may use it to
208 | // specify whether to prohibit overwriting the object with the same name
209 | // when uploading the object:
210 | //
211 | // When the x-cos-forbid-overwrite is specified as true, overwriting the object
212 | // with the same name will be prohibited.
213 | //
214 | // ref: https://www.tencentcloud.com/document/product/436/7749
215 | if args.if_not_exists() {
216 | req = req.header("x-cos-forbid-overwrite", "true")
217 | }
218 |
219 | // Set user metadata headers.
220 | if let Some(user_metadata) = args.user_metadata() {
221 | for (key, value) in user_metadata {
222 | req = req.header(format!("x-cos-meta-{key}"), value)
223 | }
224 | }
225 |
226 | let req = req.extension(Operation::Write);
227 |
228 | let req = req.body(body).map_err(new_request_build_error)?;
229 |
230 | Ok(req)
231 | }
232 |
233 | pub async fn cos_head_object(&self, path: &str, args: &OpStat) -> Result<Response<Buffer>> {
234 | let mut req = self.cos_head_object_request(path, args)?;
235 |
236 | self.sign(&mut req).await?;
237 |
238 | self.send(req).await
239 | }
240 |
241 | pub fn cos_head_object_request(&self, path: &str, args: &OpStat) -> Result<Request<Buffer>> {
242 | let p = build_abs_path(&self.root, path);
243 |
244 | let mut url = format!("{}/{}", self.endpoint, percent_encode_path(&p));
245 |
246 | let mut query_args = Vec::new();
247 | if let Some(version) = args.version() {
248 | query_args.push(format!(
249 | "{}={}",
250 | constants::COS_QUERY_VERSION_ID,
251 | percent_decode_path(version)
252 | ))
253 | }
254 | if !query_args.is_empty() {
255 | url.push_str(&format!("?{}", query_args.join("&")));
256 | }
257 |
258 | let mut req = Request::head(&url);
259 |
260 | if let Some(if_match) = args.if_match() {
261 | req = req.header(IF_MATCH, if_match);
262 | }
263 |
264 | if let Some(if_none_match) = args.if_none_match() {
265 | req = req.header(IF_NONE_MATCH, if_none_match);
266 | }
267 |
268 | let req = req.extension(Operation::Stat);
269 |
270 | let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
271 |
272 | Ok(req)
273 | }
274 |
275 | pub async fn cos_delete_object(&self, path: &str, args: &OpDelete) -> Result<Response<Buffer>> {
276 | let p = build_abs_path(&self.root, path);
277 |
278 | let mut url = format!("{}/{}", self.endpoint, percent_encode_path(&p));
279 |
280 | let mut query_args = Vec::new();
281 | if let Some(version) = args.version() {
282 | query_args.push(format!(
283 | "{}={}",
284 | constants::COS_QUERY_VERSION_ID,
285 | percent_decode_path(version)
286 | ))
287 | }
288 | if !query_args.is_empty() {
289 | url.push_str(&format!("?{}", query_args.join("&")));
290 | }
291 |
292 | let req = Request::delete(&url);
293 |
294 | let req = req.extension(Operation::Delete);
295 |
296 | let mut req = req.body(Buffer::new()).map_err(new_request_build_error)?;
297 |
298 | self.sign(&mut req).await?;
299 |
300 | self.send(req).await
301 | }
302 |
303 | pub fn cos_append_object_request(
304 | &self,
305 | path: &str,
306 | position: u64,
307 | size: u64,
308 | args: &OpWrite,
309 | body: Buffer,
310 | ) -> Result<Request<Buffer>> {
311 | let p = build_abs_path(&self.root, path);
312 | let url = format!(
313 | "{}/{}?append&position={}",
314 | self.endpoint,
315 | percent_encode_path(&p),
316 | position
317 | );
318 |
319 | let mut req = Request::post(&url);
320 |
321 | req = req.header(CONTENT_LENGTH, size);
322 |
323 | if let Some(mime) = args.content_type() {
324 | req = req.header(CONTENT_TYPE, mime);
325 | }
326 |
327 | if let Some(pos) = args.content_disposition() {
328 | req = req.header(CONTENT_DISPOSITION, pos);
329 | }
330 |
331 | if let Some(cache_control) = args.cache_control() {
332 | req = req.header(CACHE_CONTROL, cache_control)
333 | }
334 |
335 | let req = req.extension(Operation::Write);
336 |
337 | let req = req.body(body).map_err(new_request_build_error)?;
338 | Ok(req)
339 | }
340 |
341 | pub async fn cos_copy_object(&self, from: &str, to: &str) -> Result<Response<Buffer>> {
342 | let source = build_abs_path(&self.root, from);
343 | let target = build_abs_path(&self.root, to);
344 |
345 | let source = format!("/{}/{}", self.bucket, percent_encode_path(&source));
346 | let url = format!("{}/{}", self.endpoint, percent_encode_path(&target));
347 |
348 | let mut req = Request::put(&url)
349 | .extension(Operation::Copy)
350 | .header("x-cos-copy-source", &source)
351 | .body(Buffer::new())
352 | .map_err(new_request_build_error)?;
353 |
354 | self.sign(&mut req).await?;
355 |
356 | self.send(req).await
357 | }
358 |
359 | pub async fn cos_list_objects(
360 | &self,
361 | path: &str,
362 | next_marker: &str,
363 | delimiter: &str,
364 | limit: Option<usize>,
365 | ) -> Result<Response<Buffer>> {
366 | let p = build_abs_path(&self.root, path);
367 |
368 | let mut url = QueryPairsWriter::new(&self.endpoint);
369 |
370 | if !p.is_empty() {
371 | url = url.push("prefix", &percent_encode_path(&p));
372 | }
373 | if !delimiter.is_empty() {
374 | url = url.push("delimiter", delimiter);
375 | }
376 | if let Some(limit) = limit {
377 | url = url.push("max-keys", &limit.to_string());
378 | }
379 | if !next_marker.is_empty() {
380 | url = url.push("marker", next_marker);
381 | }
382 |
383 | let mut req = Request::get(url.finish())
384 | .extension(Operation::List)
385 | .body(Buffer::new())
386 | .map_err(new_request_build_error)?;
387 |
388 | self.sign(&mut req).await?;
389 |
390 | self.send(req).await
391 | }
392 |
393 | pub async fn cos_initiate_multipart_upload(
394 | &self,
395 | path: &str,
396 | args: &OpWrite,
397 | ) -> Result<Response<Buffer>> {
398 | let p = build_abs_path(&self.root, path);
399 |
400 | let url = format!("{}/{}?uploads", self.endpoint, percent_encode_path(&p));
401 |
402 | let mut req = Request::post(&url);
403 |
404 | if let Some(mime) = args.content_type() {
405 | req = req.header(CONTENT_TYPE, mime)
406 | }
407 |
408 | if let Some(content_disposition) = args.content_disposition() {
409 | req = req.header(CONTENT_DISPOSITION, content_disposition)
410 | }
411 |
412 | if let Some(cache_control) = args.cache_control() {
413 | req = req.header(CACHE_CONTROL, cache_control)
414 | }
415 |
416 | // Set user metadata headers.
417 | if let Some(user_metadata) = args.user_metadata() {
418 | for (key, value) in user_metadata {
419 | req = req.header(format!("x-cos-meta-{key}"), value)
420 | }
421 | }
422 |
423 | let req = req.extension(Operation::Write);
424 |
425 | let mut req = req.body(Buffer::new()).map_err(new_request_build_error)?;
426 |
427 | self.sign(&mut req).await?;
428 |
429 | self.send(req).await
430 | }
431 |
432 | pub async fn cos_upload_part_request(
433 | &self,
434 | path: &str,
435 | upload_id: &str,
436 | part_number: usize,
437 | size: u64,
438 | body: Buffer,
439 | ) -> Result<Response<Buffer>> {
440 | let p = build_abs_path(&self.root, path);
441 |
442 | let url = format!(
443 | "{}/{}?partNumber={}&uploadId={}",
444 | self.endpoint,
445 | percent_encode_path(&p),
446 | part_number,
447 | percent_encode_path(upload_id)
448 | );
449 |
450 | let mut req = Request::put(&url);
451 | req = req.header(CONTENT_LENGTH, size);
452 |
453 | let req = req.extension(Operation::Write);
454 |
455 | // Set body
456 | let mut req = req.body(body).map_err(new_request_build_error)?;
457 |
458 | self.sign(&mut req).await?;
459 |
460 | self.send(req).await
461 | }
462 |
463 | pub async fn cos_complete_multipart_upload(
464 | &self,
465 | path: &str,
466 | upload_id: &str,
467 | parts: Vec<CompleteMultipartUploadRequestPart>,
468 | ) -> Result<Response<Buffer>> {
469 | let p = build_abs_path(&self.root, path);
470 |
471 | let url = format!(
472 | "{}/{}?uploadId={}",
473 | self.endpoint,
474 | percent_encode_path(&p),
475 | percent_encode_path(upload_id)
476 | );
477 |
478 | let req = Request::post(&url);
479 |
480 | let content = quick_xml::se::to_string(&CompleteMultipartUploadRequest { part: parts })
481 | .map_err(new_xml_serialize_error)?;
482 | // Make sure content length has been set to avoid post with chunked encoding.
483 | let req = req.header(CONTENT_LENGTH, content.len());
484 | // Set content-type to `application/xml` to avoid mixed with form post.
485 | let req = req.header(CONTENT_TYPE, "application/xml");
486 |
487 | let req = req.extension(Operation::Write);
488 |
489 | let mut req = req
490 | .body(Buffer::from(Bytes::from(content)))
491 | .map_err(new_request_build_error)?;
492 |
493 | self.sign(&mut req).await?;
494 |
495 | self.send(req).await
496 | }
497 |
498 | /// Abort an on-going multipart upload.
499 | pub async fn cos_abort_multipart_upload(
500 | &self,
501 | path: &str,
502 | upload_id: &str,
503 | ) -> Result<Response<Buffer>> {
504 | let p = build_abs_path(&self.root, path);
505 |
506 | let url = format!(
507 | "{}/{}?uploadId={}",
508 | self.endpoint,
509 | percent_encode_path(&p),
510 | percent_encode_path(upload_id)
511 | );
512 |
513 | let mut req = Request::delete(&url)
514 | .extension(Operation::Delete)
515 | .body(Buffer::new())
516 | .map_err(new_request_build_error)?;
517 | self.sign(&mut req).await?;
518 | self.send(req).await
519 | }
520 |
521 | pub async fn cos_list_object_versions(
522 | &self,
523 | prefix: &str,
524 | delimiter: &str,
525 | limit: Option<usize>,
526 | key_marker: &str,
527 | version_id_marker: &str,
528 | ) -> Result<Response<Buffer>> {
529 | let p = build_abs_path(&self.root, prefix);
530 |
531 | let mut url = QueryPairsWriter::new(&self.endpoint);
532 | url = url.push("versions", "");
533 | if !p.is_empty() {
534 | url = url.push("prefix", &percent_encode_path(p.as_str()));
535 | }
536 | if !delimiter.is_empty() {
537 | url = url.push("delimiter", delimiter);
538 | }
539 |
540 | if let Some(limit) = limit {
541 | url = url.push("max-keys", &limit.to_string());
542 | }
543 | if !key_marker.is_empty() {
544 | url = url.push("key-marker", &percent_encode_path(key_marker));
545 | }
546 | if !version_id_marker.is_empty() {
547 | url = url.push("version-id-marker", &percent_encode_path(version_id_marker));
548 | }
549 |
550 | let mut req = Request::get(url.finish())
551 | .extension(Operation::List)
552 | .body(Buffer::new())
553 | .map_err(new_request_build_error)?;
554 |
555 | self.sign(&mut req).await?;
556 |
557 | self.send(req).await
558 | }
559 | }
560 |
561 | /// Result of CreateMultipartUpload
562 | #[derive(Default, Debug, Deserialize)]
563 | #[serde(default, rename_all = "PascalCase")]
564 | pub struct InitiateMultipartUploadResult {
565 | pub upload_id: String,
566 | }
567 |
568 | /// Request of CompleteMultipartUploadRequest
569 | #[derive(Default, Debug, Serialize)]
570 | #[serde(default, rename = "CompleteMultipartUpload", rename_all = "PascalCase")]
571 | pub struct CompleteMultipartUploadRequest {
572 | pub part: Vec<CompleteMultipartUploadRequestPart>,
573 | }
574 |
575 | #[derive(Clone, Default, Debug, Serialize)]
576 | #[serde(default, rename_all = "PascalCase")]
577 | pub struct CompleteMultipartUploadRequestPart {
578 | #[serde(rename = "PartNumber")]
579 | pub part_number: usize,
580 | /// # TODO
581 | ///
582 | /// quick-xml will do escape on `"` which leads to our serialized output is
583 | /// not the same as aws s3's example.
584 | ///
585 | /// Ideally, we could use `serialize_with` to address this (buf failed)
586 | ///
587 | /// ```ignore
588 | /// #[derive(Default, Debug, Serialize)]
589 | /// #[serde(default, rename_all = "PascalCase")]
590 | /// struct CompleteMultipartUploadRequestPart {
591 | /// #[serde(rename = "PartNumber")]
592 | /// part_number: usize,
593 | /// #[serde(rename = "ETag", serialize_with = "partial_escape")]
594 | /// etag: String,
595 | /// }
596 | ///
597 | /// fn partial_escape<S>(s: &str, ser: S) -> Result<S::Ok, S::Error>
598 | /// where
599 | /// S: serde::Serializer,
600 | /// {
601 | /// ser.serialize_str(&String::from_utf8_lossy(
602 | /// &quick_xml::escape::partial_escape(s.as_bytes()),
603 | /// ))
604 | /// }
605 | /// ```
606 | ///
607 | /// ref: <https://github.com/tafia/quick-xml/issues/362>
608 | #[serde(rename = "ETag")]
609 | pub etag: String,
610 | }
611 |
612 | /// Output of `CompleteMultipartUpload` operation
613 | #[derive(Debug, Default, Deserialize)]
614 | #[serde[default, rename_all = "PascalCase"]]
615 | pub struct CompleteMultipartUploadResult {
616 | pub location: String,
617 | pub bucket: String,
618 | pub key: String,
619 | #[serde(rename = "ETag")]
620 | pub etag: String,
621 | }
622 |
623 | #[derive(Default, Debug, Deserialize)]
624 | #[serde(default, rename_all = "PascalCase")]
625 | pub struct ListObjectsOutput {
626 | pub name: String,
627 | pub prefix: String,
628 | pub contents: Vec<ListObjectsOutputContent>,
629 | pub common_prefixes: Vec<CommonPrefix>,
630 | pub marker: String,
631 | pub next_marker: Option<String>,
632 | }
633 |
634 | #[derive(Default, Debug, Deserialize)]
635 | #[serde(default, rename_all = "PascalCase")]
636 | pub struct CommonPrefix {
637 | pub prefix: String,
638 | }
639 |
640 | #[derive(Default, Debug, Deserialize)]
641 | #[serde(default, rename_all = "PascalCase")]
642 | pub struct ListObjectsOutputContent {
643 | pub key: String,
644 | pub size: u64,
645 | }
646 |
647 | #[derive(Default, Debug, Eq, PartialEq, Deserialize)]
648 | #[serde(rename_all = "PascalCase")]
649 | pub struct OutputCommonPrefix {
650 | pub prefix: String,
651 | }
652 |
653 | /// Output of ListObjectVersions
654 | #[derive(Default, Debug, Deserialize)]
655 | #[serde(default, rename_all = "PascalCase")]
656 | pub struct ListObjectVersionsOutput {
657 | pub is_truncated: Option<bool>,
658 | pub next_key_marker: Option<String>,
659 | pub next_version_id_marker: Option<String>,
660 | pub common_prefixes: Vec<OutputCommonPrefix>,
661 | pub version: Vec<ListObjectVersionsOutputVersion>,
662 | pub delete_marker: Vec<ListObjectVersionsOutputDeleteMarker>,
663 | }
664 |
665 | #[derive(Default, Debug, Eq, PartialEq, Deserialize)]
666 | #[serde(rename_all = "PascalCase")]
667 | pub struct ListObjectVersionsOutputVersion {
668 | pub key: String,
669 | pub version_id: String,
670 | pub is_latest: bool,
671 | pub size: u64,
672 | pub last_modified: String,
673 | #[serde(rename = "ETag")]
674 | pub etag: Option<String>,
675 | }
676 |
677 | #[derive(Default, Debug, Eq, PartialEq, Deserialize)]
678 | #[serde(rename_all = "PascalCase")]
679 | pub struct ListObjectVersionsOutputDeleteMarker {
680 | pub key: String,
681 | pub version_id: String,
682 | pub is_latest: bool,
683 | pub last_modified: String,
684 | }
685 |
686 | #[cfg(test)]
687 | mod tests {
688 | use bytes::Buf;
689 |
690 | use super::*;
691 |
692 | #[test]
693 | fn test_parse_xml() {
694 | let bs = bytes::Bytes::from(
695 | r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
696 | <ListBucketResult>
697 | <Name>examplebucket</Name>
698 | <Prefix>obj</Prefix>
699 | <Marker>obj002</Marker>
700 | <NextMarker>obj004</NextMarker>
701 | <MaxKeys>1000</MaxKeys>
702 | <IsTruncated>false</IsTruncated>
703 | <Contents>
704 | <Key>obj002</Key>
705 | <LastModified>2015-07-01T02:11:19.775Z</LastModified>
706 | <ETag>"a72e382246ac83e86bd203389849e71d"</ETag>
707 | <Size>9</Size>
708 | <Owner>
709 | <ID>b4bf1b36d9ca43d984fbcb9491b6fce9</ID>
710 | </Owner>
711 | <StorageClass>STANDARD</StorageClass>
712 | </Contents>
713 | <Contents>
714 | <Key>obj003</Key>
715 | <LastModified>2015-07-01T02:11:19.775Z</LastModified>
716 | <ETag>"a72e382246ac83e86bd203389849e71d"</ETag>
717 | <Size>10</Size>
718 | <Owner>
719 | <ID>b4bf1b36d9ca43d984fbcb9491b6fce9</ID>
720 | </Owner>
721 | <StorageClass>STANDARD</StorageClass>
722 | </Contents>
723 | <CommonPrefixes>
724 | <Prefix>hello</Prefix>
725 | </CommonPrefixes>
726 | <CommonPrefixes>
727 | <Prefix>world</Prefix>
728 | </CommonPrefixes>
729 | </ListBucketResult>"#,
730 | );
731 | let out: ListObjectsOutput = quick_xml::de::from_reader(bs.reader()).expect("must success");
732 |
733 | assert_eq!(out.name, "examplebucket".to_string());
734 | assert_eq!(out.prefix, "obj".to_string());
735 | assert_eq!(out.marker, "obj002".to_string());
736 | assert_eq!(out.next_marker, Some("obj004".to_string()),);
737 | assert_eq!(
738 | out.contents
739 | .iter()
740 | .map(|v| v.key.clone())
741 | .collect::<Vec<String>>(),
742 | ["obj002", "obj003"],
743 | );
744 | assert_eq!(
745 | out.contents.iter().map(|v| v.size).collect::<Vec<u64>>(),
746 | [9, 10],
747 | );
748 | assert_eq!(
749 | out.common_prefixes
750 | .iter()
751 | .map(|v| v.prefix.clone())
752 | .collect::<Vec<String>>(),
753 | ["hello", "world"],
754 | )
755 | }
756 | }
757 |
```
--------------------------------------------------------------------------------
/bindings/nodejs/generated.js:
--------------------------------------------------------------------------------
```javascript
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing,
13 | * software distributed under the License is distributed on an
14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | * KIND, either express or implied. See the License for the
16 | * specific language governing permissions and limitations
17 | * under the License.
18 | */
19 |
20 | // prettier-ignore
21 | /* eslint-disable */
22 | // @ts-nocheck
23 | /* auto-generated by NAPI-RS */
24 |
25 | const { createRequire } = require('node:module')
26 | require = createRequire(__filename)
27 |
28 | const { readFileSync } = require('node:fs')
29 | let nativeBinding = null
30 | const loadErrors = []
31 |
32 | const isMusl = () => {
33 | let musl = false
34 | if (process.platform === 'linux') {
35 | musl = isMuslFromFilesystem()
36 | if (musl === null) {
37 | musl = isMuslFromReport()
38 | }
39 | if (musl === null) {
40 | musl = isMuslFromChildProcess()
41 | }
42 | }
43 | return musl
44 | }
45 |
46 | const isFileMusl = (f) => f.includes('libc.musl-') || f.includes('ld-musl-')
47 |
48 | const isMuslFromFilesystem = () => {
49 | try {
50 | return readFileSync('/usr/bin/ldd', 'utf-8').includes('musl')
51 | } catch {
52 | return null
53 | }
54 | }
55 |
56 | const isMuslFromReport = () => {
57 | let report = null
58 | if (typeof process.report?.getReport === 'function') {
59 | process.report.excludeNetwork = true
60 | report = process.report.getReport()
61 | }
62 | if (!report) {
63 | return null
64 | }
65 | if (report.header && report.header.glibcVersionRuntime) {
66 | return false
67 | }
68 | if (Array.isArray(report.sharedObjects)) {
69 | if (report.sharedObjects.some(isFileMusl)) {
70 | return true
71 | }
72 | }
73 | return false
74 | }
75 |
76 | const isMuslFromChildProcess = () => {
77 | try {
78 | return require('child_process').execSync('ldd --version', { encoding: 'utf8' }).includes('musl')
79 | } catch (e) {
80 | // If we reach this case, we don't know if the system is musl or not, so is better to just fallback to false
81 | return false
82 | }
83 | }
84 |
85 | function requireNative() {
86 | if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) {
87 | try {
88 | nativeBinding = require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH);
89 | } catch (err) {
90 | loadErrors.push(err)
91 | }
92 | } else if (process.platform === 'android') {
93 | if (process.arch === 'arm64') {
94 | try {
95 | return require('./opendal.android-arm64.node')
96 | } catch (e) {
97 | loadErrors.push(e)
98 | }
99 | try {
100 | const binding = require('@opendal/lib-android-arm64')
101 | const bindingPackageVersion = require('@opendal/lib-android-arm64/package.json').version
102 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
103 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
104 | }
105 | return binding
106 | } catch (e) {
107 | loadErrors.push(e)
108 | }
109 | } else if (process.arch === 'arm') {
110 | try {
111 | return require('./opendal.android-arm-eabi.node')
112 | } catch (e) {
113 | loadErrors.push(e)
114 | }
115 | try {
116 | const binding = require('@opendal/lib-android-arm-eabi')
117 | const bindingPackageVersion = require('@opendal/lib-android-arm-eabi/package.json').version
118 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
119 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
120 | }
121 | return binding
122 | } catch (e) {
123 | loadErrors.push(e)
124 | }
125 | } else {
126 | loadErrors.push(new Error(`Unsupported architecture on Android ${process.arch}`))
127 | }
128 | } else if (process.platform === 'win32') {
129 | if (process.arch === 'x64') {
130 | try {
131 | return require('./opendal.win32-x64-msvc.node')
132 | } catch (e) {
133 | loadErrors.push(e)
134 | }
135 | try {
136 | const binding = require('@opendal/lib-win32-x64-msvc')
137 | const bindingPackageVersion = require('@opendal/lib-win32-x64-msvc/package.json').version
138 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
139 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
140 | }
141 | return binding
142 | } catch (e) {
143 | loadErrors.push(e)
144 | }
145 | } else if (process.arch === 'ia32') {
146 | try {
147 | return require('./opendal.win32-ia32-msvc.node')
148 | } catch (e) {
149 | loadErrors.push(e)
150 | }
151 | try {
152 | const binding = require('@opendal/lib-win32-ia32-msvc')
153 | const bindingPackageVersion = require('@opendal/lib-win32-ia32-msvc/package.json').version
154 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
155 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
156 | }
157 | return binding
158 | } catch (e) {
159 | loadErrors.push(e)
160 | }
161 | } else if (process.arch === 'arm64') {
162 | try {
163 | return require('./opendal.win32-arm64-msvc.node')
164 | } catch (e) {
165 | loadErrors.push(e)
166 | }
167 | try {
168 | const binding = require('@opendal/lib-win32-arm64-msvc')
169 | const bindingPackageVersion = require('@opendal/lib-win32-arm64-msvc/package.json').version
170 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
171 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
172 | }
173 | return binding
174 | } catch (e) {
175 | loadErrors.push(e)
176 | }
177 | } else {
178 | loadErrors.push(new Error(`Unsupported architecture on Windows: ${process.arch}`))
179 | }
180 | } else if (process.platform === 'darwin') {
181 | try {
182 | return require('./opendal.darwin-universal.node')
183 | } catch (e) {
184 | loadErrors.push(e)
185 | }
186 | try {
187 | const binding = require('@opendal/lib-darwin-universal')
188 | const bindingPackageVersion = require('@opendal/lib-darwin-universal/package.json').version
189 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
190 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
191 | }
192 | return binding
193 | } catch (e) {
194 | loadErrors.push(e)
195 | }
196 | if (process.arch === 'x64') {
197 | try {
198 | return require('./opendal.darwin-x64.node')
199 | } catch (e) {
200 | loadErrors.push(e)
201 | }
202 | try {
203 | const binding = require('@opendal/lib-darwin-x64')
204 | const bindingPackageVersion = require('@opendal/lib-darwin-x64/package.json').version
205 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
206 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
207 | }
208 | return binding
209 | } catch (e) {
210 | loadErrors.push(e)
211 | }
212 | } else if (process.arch === 'arm64') {
213 | try {
214 | return require('./opendal.darwin-arm64.node')
215 | } catch (e) {
216 | loadErrors.push(e)
217 | }
218 | try {
219 | const binding = require('@opendal/lib-darwin-arm64')
220 | const bindingPackageVersion = require('@opendal/lib-darwin-arm64/package.json').version
221 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
222 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
223 | }
224 | return binding
225 | } catch (e) {
226 | loadErrors.push(e)
227 | }
228 | } else {
229 | loadErrors.push(new Error(`Unsupported architecture on macOS: ${process.arch}`))
230 | }
231 | } else if (process.platform === 'freebsd') {
232 | if (process.arch === 'x64') {
233 | try {
234 | return require('./opendal.freebsd-x64.node')
235 | } catch (e) {
236 | loadErrors.push(e)
237 | }
238 | try {
239 | const binding = require('@opendal/lib-freebsd-x64')
240 | const bindingPackageVersion = require('@opendal/lib-freebsd-x64/package.json').version
241 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
242 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
243 | }
244 | return binding
245 | } catch (e) {
246 | loadErrors.push(e)
247 | }
248 | } else if (process.arch === 'arm64') {
249 | try {
250 | return require('./opendal.freebsd-arm64.node')
251 | } catch (e) {
252 | loadErrors.push(e)
253 | }
254 | try {
255 | const binding = require('@opendal/lib-freebsd-arm64')
256 | const bindingPackageVersion = require('@opendal/lib-freebsd-arm64/package.json').version
257 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
258 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
259 | }
260 | return binding
261 | } catch (e) {
262 | loadErrors.push(e)
263 | }
264 | } else {
265 | loadErrors.push(new Error(`Unsupported architecture on FreeBSD: ${process.arch}`))
266 | }
267 | } else if (process.platform === 'linux') {
268 | if (process.arch === 'x64') {
269 | if (isMusl()) {
270 | try {
271 | return require('./opendal.linux-x64-musl.node')
272 | } catch (e) {
273 | loadErrors.push(e)
274 | }
275 | try {
276 | const binding = require('@opendal/lib-linux-x64-musl')
277 | const bindingPackageVersion = require('@opendal/lib-linux-x64-musl/package.json').version
278 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
279 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
280 | }
281 | return binding
282 | } catch (e) {
283 | loadErrors.push(e)
284 | }
285 | } else {
286 | try {
287 | return require('./opendal.linux-x64-gnu.node')
288 | } catch (e) {
289 | loadErrors.push(e)
290 | }
291 | try {
292 | const binding = require('@opendal/lib-linux-x64-gnu')
293 | const bindingPackageVersion = require('@opendal/lib-linux-x64-gnu/package.json').version
294 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
295 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
296 | }
297 | return binding
298 | } catch (e) {
299 | loadErrors.push(e)
300 | }
301 | }
302 | } else if (process.arch === 'arm64') {
303 | if (isMusl()) {
304 | try {
305 | return require('./opendal.linux-arm64-musl.node')
306 | } catch (e) {
307 | loadErrors.push(e)
308 | }
309 | try {
310 | const binding = require('@opendal/lib-linux-arm64-musl')
311 | const bindingPackageVersion = require('@opendal/lib-linux-arm64-musl/package.json').version
312 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
313 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
314 | }
315 | return binding
316 | } catch (e) {
317 | loadErrors.push(e)
318 | }
319 | } else {
320 | try {
321 | return require('./opendal.linux-arm64-gnu.node')
322 | } catch (e) {
323 | loadErrors.push(e)
324 | }
325 | try {
326 | const binding = require('@opendal/lib-linux-arm64-gnu')
327 | const bindingPackageVersion = require('@opendal/lib-linux-arm64-gnu/package.json').version
328 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
329 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
330 | }
331 | return binding
332 | } catch (e) {
333 | loadErrors.push(e)
334 | }
335 | }
336 | } else if (process.arch === 'arm') {
337 | if (isMusl()) {
338 | try {
339 | return require('./opendal.linux-arm-musleabihf.node')
340 | } catch (e) {
341 | loadErrors.push(e)
342 | }
343 | try {
344 | const binding = require('@opendal/lib-linux-arm-musleabihf')
345 | const bindingPackageVersion = require('@opendal/lib-linux-arm-musleabihf/package.json').version
346 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
347 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
348 | }
349 | return binding
350 | } catch (e) {
351 | loadErrors.push(e)
352 | }
353 | } else {
354 | try {
355 | return require('./opendal.linux-arm-gnueabihf.node')
356 | } catch (e) {
357 | loadErrors.push(e)
358 | }
359 | try {
360 | const binding = require('@opendal/lib-linux-arm-gnueabihf')
361 | const bindingPackageVersion = require('@opendal/lib-linux-arm-gnueabihf/package.json').version
362 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
363 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
364 | }
365 | return binding
366 | } catch (e) {
367 | loadErrors.push(e)
368 | }
369 | }
370 | } else if (process.arch === 'riscv64') {
371 | if (isMusl()) {
372 | try {
373 | return require('./opendal.linux-riscv64-musl.node')
374 | } catch (e) {
375 | loadErrors.push(e)
376 | }
377 | try {
378 | const binding = require('@opendal/lib-linux-riscv64-musl')
379 | const bindingPackageVersion = require('@opendal/lib-linux-riscv64-musl/package.json').version
380 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
381 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
382 | }
383 | return binding
384 | } catch (e) {
385 | loadErrors.push(e)
386 | }
387 | } else {
388 | try {
389 | return require('./opendal.linux-riscv64-gnu.node')
390 | } catch (e) {
391 | loadErrors.push(e)
392 | }
393 | try {
394 | const binding = require('@opendal/lib-linux-riscv64-gnu')
395 | const bindingPackageVersion = require('@opendal/lib-linux-riscv64-gnu/package.json').version
396 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
397 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
398 | }
399 | return binding
400 | } catch (e) {
401 | loadErrors.push(e)
402 | }
403 | }
404 | } else if (process.arch === 'ppc64') {
405 | try {
406 | return require('./opendal.linux-ppc64-gnu.node')
407 | } catch (e) {
408 | loadErrors.push(e)
409 | }
410 | try {
411 | const binding = require('@opendal/lib-linux-ppc64-gnu')
412 | const bindingPackageVersion = require('@opendal/lib-linux-ppc64-gnu/package.json').version
413 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
414 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
415 | }
416 | return binding
417 | } catch (e) {
418 | loadErrors.push(e)
419 | }
420 | } else if (process.arch === 's390x') {
421 | try {
422 | return require('./opendal.linux-s390x-gnu.node')
423 | } catch (e) {
424 | loadErrors.push(e)
425 | }
426 | try {
427 | const binding = require('@opendal/lib-linux-s390x-gnu')
428 | const bindingPackageVersion = require('@opendal/lib-linux-s390x-gnu/package.json').version
429 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
430 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
431 | }
432 | return binding
433 | } catch (e) {
434 | loadErrors.push(e)
435 | }
436 | } else {
437 | loadErrors.push(new Error(`Unsupported architecture on Linux: ${process.arch}`))
438 | }
439 | } else if (process.platform === 'openharmony') {
440 | if (process.arch === 'arm64') {
441 | try {
442 | return require('./opendal.openharmony-arm64.node')
443 | } catch (e) {
444 | loadErrors.push(e)
445 | }
446 | try {
447 | const binding = require('@opendal/lib-openharmony-arm64')
448 | const bindingPackageVersion = require('@opendal/lib-openharmony-arm64/package.json').version
449 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
450 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
451 | }
452 | return binding
453 | } catch (e) {
454 | loadErrors.push(e)
455 | }
456 | } else if (process.arch === 'x64') {
457 | try {
458 | return require('./opendal.openharmony-x64.node')
459 | } catch (e) {
460 | loadErrors.push(e)
461 | }
462 | try {
463 | const binding = require('@opendal/lib-openharmony-x64')
464 | const bindingPackageVersion = require('@opendal/lib-openharmony-x64/package.json').version
465 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
466 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
467 | }
468 | return binding
469 | } catch (e) {
470 | loadErrors.push(e)
471 | }
472 | } else if (process.arch === 'arm') {
473 | try {
474 | return require('./opendal.openharmony-arm.node')
475 | } catch (e) {
476 | loadErrors.push(e)
477 | }
478 | try {
479 | const binding = require('@opendal/lib-openharmony-arm')
480 | const bindingPackageVersion = require('@opendal/lib-openharmony-arm/package.json').version
481 | if (bindingPackageVersion !== '0.49.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
482 | throw new Error(`Native binding package version mismatch, expected 0.49.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
483 | }
484 | return binding
485 | } catch (e) {
486 | loadErrors.push(e)
487 | }
488 | } else {
489 | loadErrors.push(new Error(`Unsupported architecture on OpenHarmony: ${process.arch}`))
490 | }
491 | } else {
492 | loadErrors.push(new Error(`Unsupported OS: ${process.platform}, architecture: ${process.arch}`))
493 | }
494 | }
495 |
496 | nativeBinding = requireNative()
497 |
498 | if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
499 | try {
500 | nativeBinding = require('./opendal.wasi.cjs')
501 | } catch (err) {
502 | if (process.env.NAPI_RS_FORCE_WASI) {
503 | loadErrors.push(err)
504 | }
505 | }
506 | if (!nativeBinding) {
507 | try {
508 | nativeBinding = require('@opendal/lib-wasm32-wasi')
509 | } catch (err) {
510 | if (process.env.NAPI_RS_FORCE_WASI) {
511 | loadErrors.push(err)
512 | }
513 | }
514 | }
515 | }
516 |
517 | if (!nativeBinding) {
518 | if (loadErrors.length > 0) {
519 | throw new Error(
520 | `Cannot find native binding. ` +
521 | `npm has a bug related to optional dependencies (https://github.com/npm/cli/issues/4828). ` +
522 | 'Please try `npm i` again after removing both package-lock.json and node_modules directory.',
523 | { cause: loadErrors }
524 | )
525 | }
526 | throw new Error(`Failed to load native binding`)
527 | }
528 |
529 | module.exports = nativeBinding
530 | module.exports.BlockingLister = nativeBinding.BlockingLister
531 | module.exports.BlockingReader = nativeBinding.BlockingReader
532 | module.exports.BlockingWriter = nativeBinding.BlockingWriter
533 | module.exports.Capability = nativeBinding.Capability
534 | module.exports.ConcurrentLimitLayer = nativeBinding.ConcurrentLimitLayer
535 | module.exports.Entry = nativeBinding.Entry
536 | module.exports.Layer = nativeBinding.Layer
537 | module.exports.Lister = nativeBinding.Lister
538 | module.exports.LoggingLayer = nativeBinding.LoggingLayer
539 | module.exports.Metadata = nativeBinding.Metadata
540 | module.exports.Operator = nativeBinding.Operator
541 | module.exports.Reader = nativeBinding.Reader
542 | module.exports.RetryLayer = nativeBinding.RetryLayer
543 | module.exports.ThrottleLayer = nativeBinding.ThrottleLayer
544 | module.exports.TimeoutLayer = nativeBinding.TimeoutLayer
545 | module.exports.Writer = nativeBinding.Writer
546 | module.exports.EntryMode = nativeBinding.EntryMode
547 |
```
--------------------------------------------------------------------------------
/core/core/src/types/options.rs:
--------------------------------------------------------------------------------
```rust
1 | // Licensed to the Apache Software Foundation (ASF) under one
2 | // or more contributor license agreements. See the NOTICE file
3 | // distributed with this work for additional information
4 | // regarding copyright ownership. The ASF licenses this file
5 | // to you under the Apache License, Version 2.0 (the
6 | // "License"); you may not use this file except in compliance
7 | // with the License. You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing,
12 | // software distributed under the License is distributed on an
13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 | // KIND, either express or implied. See the License for the
15 | // specific language governing permissions and limitations
16 | // under the License.
17 |
18 | //! Options module provides options definitions for operations.
19 |
20 | use crate::raw::{BytesRange, Timestamp};
21 | use std::collections::HashMap;
22 |
23 | /// Options for delete operations.
24 | #[derive(Debug, Clone, Default, Eq, PartialEq)]
25 | pub struct DeleteOptions {
26 | /// The version of the file to delete.
27 | pub version: Option<String>,
28 | /// Whether to delete the target recursively.
29 | ///
30 | /// - If `false`, behaves like the traditional single-object delete.
31 | /// - If `true`, all entries under the path (or sharing the prefix for file-like paths)
32 | /// will be removed.
33 | pub recursive: bool,
34 | }
35 |
36 | /// Options for list operations.
37 | ///
38 | /// # Groups
39 | /// - Traversal: `recursive`.
40 | /// - Pagination: `limit`, `start_after`.
41 | /// - Versioning: `versions`, `deleted` (effective on version-aware backends).
42 |
43 | #[derive(Debug, Clone, Default, Eq, PartialEq)]
44 | pub struct ListOptions {
45 | /// Maximum results per request (backend hint) to control memory and throttling.
46 | pub limit: Option<usize>,
47 | /// The start_after passes to underlying service to specify the specified key
48 | /// to start listing from.
49 | pub start_after: Option<String>,
50 | /// Whether to list recursively under the prefix; default `false`.
51 | pub recursive: bool,
52 | /// Include object versions when supported by the backend; default `false`.
53 | pub versions: bool,
54 | /// Include delete markers when supported by version-aware backends; default `false`.
55 | pub deleted: bool,
56 | }
57 |
58 | /// Options for read operations.
59 | #[derive(Debug, Clone, Default, Eq, PartialEq)]
60 | pub struct ReadOptions {
61 | /// Set `range` for this operation.
62 | ///
63 | /// If we have a file with size `n`.
64 | ///
65 | /// - `..` means read bytes in range `[0, n)` of file.
66 | /// - `0..1024` and `..1024` means read bytes in range `[0, 1024)` of file
67 | /// - `1024..` means read bytes in range `[1024, n)` of file
68 | ///
69 | /// The type implements `From<RangeBounds<u64>>`, so users can use `(1024..).into()` instead.
70 | pub range: BytesRange,
71 | /// Set `version` for this operation.
72 | ///
73 | /// This option can be used to retrieve the data of a specified version of the given path.
74 | ///
75 | /// If the version doesn't exist, an error with kind [`ErrorKind::NotFound`] will be returned.
76 | pub version: Option<String>,
77 |
78 | /// Set `if_match` for this operation.
79 | ///
80 | /// This option can be used to check if the file's `ETag` matches the given `ETag`.
81 | ///
82 | /// If file exists and it's etag doesn't match, an error with kind [`ErrorKind::ConditionNotMatch`]
83 | /// will be returned.
84 | pub if_match: Option<String>,
85 | /// Set `if_none_match` for this operation.
86 | ///
87 | /// This option can be used to check if the file's `ETag` doesn't match the given `ETag`.
88 | ///
89 | /// If file exists and it's etag match, an error with kind [`ErrorKind::ConditionNotMatch`]
90 | /// will be returned.
91 | pub if_none_match: Option<String>,
92 | /// Set `if_modified_since` for this operation.
93 | ///
94 | /// This option can be used to check if the file has been modified since the given timestamp.
95 | ///
96 | /// If file exists and it hasn't been modified since the specified time, an error with kind
97 | /// [`ErrorKind::ConditionNotMatch`] will be returned.
98 | pub if_modified_since: Option<Timestamp>,
99 | /// Set `if_unmodified_since` for this operation.
100 | ///
101 | /// This feature can be used to check if the file hasn't been modified since the given timestamp.
102 | ///
103 | /// If file exists and it has been modified since the specified time, an error with kind
104 | /// [`ErrorKind::ConditionNotMatch`] will be returned.
105 | pub if_unmodified_since: Option<Timestamp>,
106 |
107 | /// Set `concurrent` for the operation.
108 | ///
109 | /// OpenDAL by default to read file without concurrent. This is not efficient for cases when users
110 | /// read large chunks of data. By setting `concurrent`, opendal will reading files concurrently
111 | /// on support storage services.
112 | ///
113 | /// By setting `concurrent`, opendal will fetch chunks concurrently with
114 | /// the give chunk size.
115 | ///
116 | /// Refer to [`crate::docs::performance`] for more details.
117 | pub concurrent: usize,
118 | /// Set `chunk` for the operation.
119 | ///
120 | /// OpenDAL will use services' preferred chunk size by default. Users can set chunk based on their own needs.
121 | ///
122 | /// Refer to [`crate::docs::performance`] for more details.
123 | pub chunk: Option<usize>,
124 | /// Controls the optimization strategy for range reads in [`Reader::fetch`].
125 | ///
126 | /// When performing range reads, if the gap between two requested ranges is smaller than
127 | /// the configured `gap` size, OpenDAL will merge these ranges into a single read request
128 | /// and discard the unrequested data in between. This helps reduce the number of API calls
129 | /// to remote storage services.
130 | ///
131 | /// This optimization is particularly useful when performing multiple small range reads
132 | /// that are close to each other, as it reduces the overhead of multiple network requests
133 | /// at the cost of transferring some additional data.
134 | ///
135 | /// Refer to [`crate::docs::performance`] for more details.
136 | pub gap: Option<usize>,
137 |
138 | /// Specify the content-type header that should be sent back by the operation.
139 | ///
140 | /// This option is only meaningful when used along with presign.
141 | pub override_content_type: Option<String>,
142 | /// Specify the `cache-control` header that should be sent back by the operation.
143 | ///
144 | /// This option is only meaningful when used along with presign.
145 | pub override_cache_control: Option<String>,
146 | /// Specify the `content-disposition` header that should be sent back by the operation.
147 | ///
148 | /// This option is only meaningful when used along with presign.
149 | pub override_content_disposition: Option<String>,
150 | }
151 |
152 | /// Options for reader operations.
153 | #[derive(Debug, Clone, Default, Eq, PartialEq)]
154 | pub struct ReaderOptions {
155 | /// Set `version` for this operation.
156 | ///
157 | /// This option can be used to retrieve the data of a specified version of the given path.
158 | ///
159 | /// If the version doesn't exist, an error with kind [`ErrorKind::NotFound`] will be returned.
160 | pub version: Option<String>,
161 |
162 | /// Set `if_match` for this operation.
163 | ///
164 | /// This option can be used to check if the file's `ETag` matches the given `ETag`.
165 | ///
166 | /// If file exists and it's etag doesn't match, an error with kind [`ErrorKind::ConditionNotMatch`]
167 | /// will be returned.
168 | pub if_match: Option<String>,
169 | /// Set `if_none_match` for this operation.
170 | ///
171 | /// This option can be used to check if the file's `ETag` doesn't match the given `ETag`.
172 | ///
173 | /// If file exists and it's etag match, an error with kind [`ErrorKind::ConditionNotMatch`]
174 | /// will be returned.
175 | pub if_none_match: Option<String>,
176 | /// Set `if_modified_since` for this operation.
177 | ///
178 | /// This option can be used to check if the file has been modified since the given timestamp.
179 | ///
180 | /// If file exists and it hasn't been modified since the specified time, an error with kind
181 | /// [`ErrorKind::ConditionNotMatch`] will be returned.
182 | pub if_modified_since: Option<Timestamp>,
183 | /// Set `if_unmodified_since` for this operation.
184 | ///
185 | /// This feature can be used to check if the file hasn't been modified since the given timestamp.
186 | ///
187 | /// If file exists and it has been modified since the specified time, an error with kind
188 | /// [`ErrorKind::ConditionNotMatch`] will be returned.
189 | pub if_unmodified_since: Option<Timestamp>,
190 |
191 | /// Set `concurrent` for the operation.
192 | ///
193 | /// OpenDAL by default to read file without concurrent. This is not efficient for cases when users
194 | /// read large chunks of data. By setting `concurrent`, opendal will reading files concurrently
195 | /// on support storage services.
196 | ///
197 | /// By setting `concurrent`, opendal will fetch chunks concurrently with
198 | /// the give chunk size.
199 | ///
200 | /// Refer to [`crate::docs::performance`] for more details.
201 | pub concurrent: usize,
202 | /// Set `chunk` for the operation.
203 | ///
204 | /// OpenDAL will use services' preferred chunk size by default. Users can set chunk based on their own needs.
205 | ///
206 | /// Refer to [`crate::docs::performance`] for more details.
207 | pub chunk: Option<usize>,
208 | /// Controls the optimization strategy for range reads in [`Reader::fetch`].
209 | ///
210 | /// When performing range reads, if the gap between two requested ranges is smaller than
211 | /// the configured `gap` size, OpenDAL will merge these ranges into a single read request
212 | /// and discard the unrequested data in between. This helps reduce the number of API calls
213 | /// to remote storage services.
214 | ///
215 | /// This optimization is particularly useful when performing multiple small range reads
216 | /// that are close to each other, as it reduces the overhead of multiple network requests
217 | /// at the cost of transferring some additional data.
218 | ///
219 | /// Refer to [`crate::docs::performance`] for more details.
220 | pub gap: Option<usize>,
221 | /// Controls the number of prefetched bytes ranges that can be buffered in memory
222 | /// during concurrent reading.
223 | ///
224 | /// When performing concurrent reads with `Reader`, this option limits how many
225 | /// completed-but-not-yet-read chunks can be buffered. Once the number of buffered
226 | /// chunks reaches this limit, no new read tasks will be spawned until some of the
227 | /// buffered chunks are consumed.
228 | ///
229 | /// - Default value: 0 (no prefetching, strict back-pressure control)
230 | /// - Set to a higher value to allow more aggressive prefetching at the cost of memory
231 | ///
232 | /// This option helps prevent memory exhaustion when reading large files with high
233 | /// concurrency settings.
234 | pub prefetch: usize,
235 | }
236 |
237 | /// Options for stat operations.
238 | #[derive(Debug, Clone, Default, Eq, PartialEq)]
239 | pub struct StatOptions {
240 | /// Set `version` for this operation.
241 | ///
242 | /// This options can be used to retrieve the data of a specified version of the given path.
243 | ///
244 | /// If the version doesn't exist, an error with kind [`ErrorKind::NotFound`] will be returned.
245 | pub version: Option<String>,
246 |
247 | /// Set `if_match` for this operation.
248 | ///
249 | /// This option can be used to check if the file's `ETag` matches the given `ETag`.
250 | ///
251 | /// If file exists and it's etag doesn't match, an error with kind [`ErrorKind::ConditionNotMatch`]
252 | /// will be returned.
253 | pub if_match: Option<String>,
254 | /// Set `if_none_match` for this operation.
255 | ///
256 | /// This option can be used to check if the file's `ETag` doesn't match the given `ETag`.
257 | ///
258 | /// If file exists and it's etag match, an error with kind [`ErrorKind::ConditionNotMatch`]
259 | /// will be returned.
260 | pub if_none_match: Option<String>,
261 | /// Set `if_modified_since` for this operation.
262 | ///
263 | /// This option can be used to check if the file has been modified since the given timestamp.
264 | ///
265 | /// If file exists and it hasn't been modified since the specified time, an error with kind
266 | /// [`ErrorKind::ConditionNotMatch`] will be returned.
267 | pub if_modified_since: Option<Timestamp>,
268 | /// Set `if_unmodified_since` for this operation.
269 | ///
270 | /// This feature can be used to check if the file hasn't been modified since the given timestamp.
271 | ///
272 | /// If file exists and it has been modified since the specified time, an error with kind
273 | /// [`ErrorKind::ConditionNotMatch`] will be returned.
274 | pub if_unmodified_since: Option<Timestamp>,
275 |
276 | /// Specify the content-type header that should be sent back by the operation.
277 | ///
278 | /// This option is only meaningful when used along with presign.
279 | pub override_content_type: Option<String>,
280 | /// Specify the `cache-control` header that should be sent back by the operation.
281 | ///
282 | /// This option is only meaningful when used along with presign.
283 | pub override_cache_control: Option<String>,
284 | /// Specify the `content-disposition` header that should be sent back by the operation.
285 | ///
286 | /// This option is only meaningful when used along with presign.
287 | pub override_content_disposition: Option<String>,
288 | }
289 |
290 | /// Options for write operations.
291 | #[derive(Debug, Clone, Default, Eq, PartialEq)]
292 | pub struct WriteOptions {
293 | /// Sets append mode for this operation.
294 | ///
295 | /// ### Capability
296 | ///
297 | /// Check [`Capability::write_can_append`] before using this option.
298 | ///
299 | /// ### Behavior
300 | ///
301 | /// - By default, write operations overwrite existing files
302 | /// - When append is set to true:
303 | /// - New data will be appended to the end of existing file
304 | /// - If file doesn't exist, it will be created
305 | /// - If not supported, will return an error
306 | ///
307 | /// This operation allows adding data to existing files instead of overwriting them.
308 | pub append: bool,
309 |
310 | /// Sets Cache-Control header for this write operation.
311 | ///
312 | /// ### Capability
313 | ///
314 | /// Check [`Capability::write_with_cache_control`] before using this feature.
315 | ///
316 | /// ### Behavior
317 | ///
318 | /// - If supported, sets Cache-Control as system metadata on the target file
319 | /// - The value should follow HTTP Cache-Control header format
320 | /// - If not supported, the value will be ignored
321 | ///
322 | /// This operation allows controlling caching behavior for the written content.
323 | ///
324 | /// ### Use Cases
325 | ///
326 | /// - Setting browser cache duration
327 | /// - Configuring CDN behavior
328 | /// - Optimizing content delivery
329 | /// - Managing cache invalidation
330 | ///
331 | /// ### References
332 | ///
333 | /// - [MDN Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control)
334 | /// - [RFC 7234 Section 5.2](https://tools.ietf.org/html/rfc7234#section-5.2)
335 | pub cache_control: Option<String>,
336 | /// Sets `Content-Type` header for this write operation.
337 | ///
338 | /// ## Capability
339 | ///
340 | /// Check [`Capability::write_with_content_type`] before using this feature.
341 | ///
342 | /// ### Behavior
343 | ///
344 | /// - If supported, sets Content-Type as system metadata on the target file
345 | /// - The value should follow MIME type format (e.g. "text/plain", "image/jpeg")
346 | /// - If not supported, the value will be ignored
347 | ///
348 | /// This operation allows specifying the media type of the content being written.
349 | pub content_type: Option<String>,
350 | /// Sets Content-Disposition header for this write request.
351 | ///
352 | /// ### Capability
353 | ///
354 | /// Check [`Capability::write_with_content_disposition`] before using this feature.
355 | ///
356 | /// ### Behavior
357 | ///
358 | /// - If supported, sets Content-Disposition as system metadata on the target file
359 | /// - The value should follow HTTP Content-Disposition header format
360 | /// - Common values include:
361 | /// - `inline` - Content displayed within browser
362 | /// - `attachment` - Content downloaded as file
363 | /// - `attachment; filename="example.jpg"` - Downloaded with specified filename
364 | /// - If not supported, the value will be ignored
365 | ///
366 | /// This operation allows controlling how the content should be displayed or downloaded.
367 | pub content_disposition: Option<String>,
368 | /// Sets Content-Encoding header for this write request.
369 | ///
370 | /// ### Capability
371 | ///
372 | /// Check [`Capability::write_with_content_encoding`] before using this feature.
373 | ///
374 | /// ### Behavior
375 | ///
376 | /// - If supported, sets Content-Encoding as system metadata on the target file
377 | /// - The value should follow HTTP Content-Encoding header format
378 | /// - Common values include:
379 | /// - `gzip` - Content encoded using gzip compression
380 | /// - `deflate` - Content encoded using deflate compression
381 | /// - `br` - Content encoded using Brotli compression
382 | /// - `identity` - No encoding applied (default value)
383 | /// - If not supported, the value will be ignored
384 | ///
385 | /// This operation allows specifying the encoding applied to the content being written.
386 | pub content_encoding: Option<String>,
387 | /// Sets user metadata for this write request.
388 | ///
389 | /// ### Capability
390 | ///
391 | /// Check [`Capability::write_with_user_metadata`] before using this feature.
392 | ///
393 | /// ### Behavior
394 | ///
395 | /// - If supported, the user metadata will be attached to the object during write
396 | /// - Accepts key-value pairs where both key and value are strings
397 | /// - Keys are case-insensitive in most services
398 | /// - Services may have limitations for user metadata, for example:
399 | /// - Key length is typically limited (e.g., 1024 bytes)
400 | /// - Value length is typically limited (e.g., 4096 bytes)
401 | /// - Total metadata size might be limited
402 | /// - Some characters might be forbidden in keys
403 | /// - If not supported, the metadata will be ignored
404 | ///
405 | /// User metadata provides a way to attach custom metadata to objects during write operations.
406 | /// This metadata can be retrieved later when reading the object.
407 | pub user_metadata: Option<HashMap<String, String>>,
408 |
409 | /// Sets If-Match header for this write request.
410 | ///
411 | /// ### Capability
412 | ///
413 | /// Check [`Capability::write_with_if_match`] before using this feature.
414 | ///
415 | /// ### Behavior
416 | ///
417 | /// - If supported, the write operation will only succeed if the target's ETag matches the specified value
418 | /// - The value should be a valid ETag string
419 | /// - Common values include:
420 | /// - A specific ETag value like `"686897696a7c876b7e"`
421 | /// - `*` - Matches any existing resource
422 | /// - If not supported, the value will be ignored
423 | ///
424 | /// This operation provides conditional write functionality based on ETag matching,
425 | /// helping prevent unintended overwrites in concurrent scenarios.
426 | pub if_match: Option<String>,
427 | /// Sets If-None-Match header for this write request.
428 | ///
429 | /// Note: Certain services, like `s3`, support `if_not_exists` but not `if_none_match`.
430 | /// Use `if_not_exists` if you only want to check whether a file exists.
431 | ///
432 | /// ### Capability
433 | ///
434 | /// Check [`Capability::write_with_if_none_match`] before using this feature.
435 | ///
436 | /// ### Behavior
437 | ///
438 | /// - If supported, the write operation will only succeed if the target's ETag does not match the specified value
439 | /// - The value should be a valid ETag string
440 | /// - Common values include:
441 | /// - A specific ETag value like `"686897696a7c876b7e"`
442 | /// - `*` - Matches if the resource does not exist
443 | /// - If not supported, the value will be ignored
444 | ///
445 | /// This operation provides conditional write functionality based on ETag non-matching,
446 | /// useful for preventing overwriting existing resources or ensuring unique writes.
447 | pub if_none_match: Option<String>,
448 | /// Sets the condition that write operation will succeed only if target does not exist.
449 | ///
450 | /// ### Capability
451 | ///
452 | /// Check [`Capability::write_with_if_not_exists`] before using this feature.
453 | ///
454 | /// ### Behavior
455 | ///
456 | /// - If supported, the write operation will only succeed if the target path does not exist
457 | /// - Will return error if target already exists
458 | /// - If not supported, the value will be ignored
459 | ///
460 | /// This operation provides a way to ensure write operations only create new resources
461 | /// without overwriting existing ones, useful for implementing "create if not exists" logic.
462 | pub if_not_exists: bool,
463 |
464 | /// Sets concurrent write operations for this writer.
465 | ///
466 | /// ## Behavior
467 | ///
468 | /// - By default, OpenDAL writes files sequentially
469 | /// - When concurrent is set:
470 | /// - Multiple write operations can execute in parallel
471 | /// - Write operations return immediately without waiting if tasks space are available
472 | /// - Close operation ensures all writes complete in order
473 | /// - Memory usage increases with concurrency level
474 | /// - If not supported, falls back to sequential writes
475 | ///
476 | /// This feature significantly improves performance when:
477 | /// - Writing large files
478 | /// - Network latency is high
479 | /// - Storage service supports concurrent uploads like multipart uploads
480 | ///
481 | /// ## Performance Impact
482 | ///
483 | /// Setting appropriate concurrency can:
484 | /// - Increase write throughput
485 | /// - Reduce total write time
486 | /// - Better utilize available bandwidth
487 | /// - Trade memory for performance
488 | pub concurrent: usize,
489 | /// Sets chunk size for buffered writes.
490 | ///
491 | /// ### Capability
492 | ///
493 | /// Check [`Capability::write_multi_min_size`] and [`Capability::write_multi_max_size`] for size limits.
494 | ///
495 | /// ### Behavior
496 | ///
497 | /// - By default, OpenDAL sets optimal chunk size based on service capabilities
498 | /// - When chunk size is set:
499 | /// - Data will be buffered until reaching chunk size
500 | /// - One API call will be made per chunk
501 | /// - Last chunk may be smaller than chunk size
502 | /// - Important considerations:
503 | /// - Some services require minimum chunk sizes (e.g. S3's EntityTooSmall error)
504 | /// - Smaller chunks increase API calls and costs
505 | /// - Larger chunks increase memory usage, but improve performance and reduce costs
506 | ///
507 | /// ### Performance Impact
508 | ///
509 | /// Setting appropriate chunk size can:
510 | /// - Reduce number of API calls
511 | /// - Improve overall throughput
512 | /// - Lower operation costs
513 | /// - Better utilize network bandwidth
514 | pub chunk: Option<usize>,
515 | }
516 |
517 | /// Options for copy operations.
518 | #[derive(Debug, Clone, Default, Eq, PartialEq)]
519 | pub struct CopyOptions {
520 | /// Sets the condition that copy operation will succeed only if target does not exist.
521 | ///
522 | /// ### Capability
523 | ///
524 | /// Check [`Capability::copy_with_if_not_exists`] before using this feature.
525 | ///
526 | /// ### Behavior
527 | ///
528 | /// - If supported, the copy operation will only succeed if the target path does not exist
529 | /// - Will return error if target already exists
530 | /// - If not supported, the value will be ignored
531 | ///
532 | /// This operation provides a way to ensure copy operations only create new resources
533 | /// without overwriting existing ones, useful for implementing "copy if not exists" logic.
534 | pub if_not_exists: bool,
535 | }
536 |
```