This is page 1 of 55. Use http://codebase.md/apache/opendal?page={x} to view the full context.
# Directory Structure
```
├── .asf.yaml
├── .config
│ └── nextest.toml
├── .devcontainer
│ ├── devcontainer.json
│ └── post_create.sh
├── .editorconfig
├── .env.example
├── .gitattributes
├── .github
│ ├── actions
│ │ ├── fuzz_test
│ │ │ └── action.yaml
│ │ ├── setup
│ │ │ └── action.yaml
│ │ ├── setup-hadoop
│ │ │ └── action.yaml
│ │ ├── setup-ocaml
│ │ │ └── action.yaml
│ │ ├── test_behavior_binding_c
│ │ │ └── action.yaml
│ │ ├── test_behavior_binding_cpp
│ │ │ └── action.yaml
│ │ ├── test_behavior_binding_go
│ │ │ └── action.yaml
│ │ ├── test_behavior_binding_java
│ │ │ └── action.yaml
│ │ ├── test_behavior_binding_nodejs
│ │ │ └── action.yaml
│ │ ├── test_behavior_binding_python
│ │ │ └── action.yaml
│ │ ├── test_behavior_core
│ │ │ └── action.yaml
│ │ └── test_behavior_integration_object_store
│ │ └── action.yml
│ ├── CODEOWNERS
│ ├── dependabot.yml
│ ├── ISSUE_TEMPLATE
│ │ ├── 1-bug-report.yml
│ │ ├── 2-feature-request.yml
│ │ ├── 3-new-release.md
│ │ └── config.yml
│ ├── pull_request_template.md
│ ├── release.yml
│ ├── scripts
│ │ ├── test_behavior
│ │ │ ├── __init__.py
│ │ │ ├── plan.py
│ │ │ └── test_plan.py
│ │ ├── test_go_binding
│ │ │ ├── generate_test_scheme.py
│ │ │ └── matrix.yaml
│ │ └── weekly_update
│ │ ├── .gitignore
│ │ ├── .python-version
│ │ ├── main.py
│ │ ├── pyproject.toml
│ │ ├── README.md
│ │ └── uv.lock
│ ├── services
│ │ ├── aliyun_drive
│ │ │ └── aliyun_drive
│ │ │ └── disable_action.yml
│ │ ├── alluxio
│ │ │ └── alluxio
│ │ │ └── action.yml
│ │ ├── azblob
│ │ │ ├── azure_azblob
│ │ │ │ └── action.yml
│ │ │ └── azurite_azblob
│ │ │ └── action.yml
│ │ ├── azdls
│ │ │ └── azdls
│ │ │ └── action.yml
│ │ ├── azfile
│ │ │ └── azfile
│ │ │ └── action.yml
│ │ ├── b2
│ │ │ └── b2
│ │ │ └── action.yml
│ │ ├── cacache
│ │ │ └── cacache
│ │ │ └── action.yml
│ │ ├── compfs
│ │ │ └── compfs
│ │ │ └── action.yml
│ │ ├── cos
│ │ │ └── cos
│ │ │ └── action.yml
│ │ ├── dashmap
│ │ │ └── dashmap
│ │ │ └── action.yml
│ │ ├── dropbox
│ │ │ └── dropbox
│ │ │ └── disable_action.yml
│ │ ├── etcd
│ │ │ ├── etcd
│ │ │ │ └── action.yml
│ │ │ ├── etcd-cluster
│ │ │ │ └── action.yml
│ │ │ └── etcd-tls
│ │ │ └── action.yml
│ │ ├── fs
│ │ │ └── local_fs
│ │ │ └── action.yml
│ │ ├── ftp
│ │ │ └── vsftpd
│ │ │ └── disable_action.yml
│ │ ├── gcs
│ │ │ ├── gcs
│ │ │ │ └── action.yml
│ │ │ └── gcs_with_default_storage_class
│ │ │ └── action.yml
│ │ ├── gdrive
│ │ │ └── gdrive
│ │ │ └── action.yml
│ │ ├── gridfs
│ │ │ ├── gridfs
│ │ │ │ └── action.yml
│ │ │ └── gridfs_with_basic_auth
│ │ │ └── action.yml
│ │ ├── hdfs
│ │ │ ├── hdfs_cluster
│ │ │ │ └── action.yml
│ │ │ ├── hdfs_cluster_with_atomic_write_dir
│ │ │ │ └── action.yml
│ │ │ ├── hdfs_default
│ │ │ │ └── action.yml
│ │ │ ├── hdfs_default_gcs
│ │ │ │ └── action.yml
│ │ │ ├── hdfs_default_on_azurite_azblob
│ │ │ │ └── action.yml
│ │ │ ├── hdfs_default_on_minio_s3
│ │ │ │ └── action.yml
│ │ │ └── hdfs_default_with_atomic_write_dir
│ │ │ └── action.yml
│ │ ├── hdfs_native
│ │ │ └── hdfs_native_cluster
│ │ │ └── action.yml
│ │ ├── http
│ │ │ ├── caddy
│ │ │ │ └── action.yml
│ │ │ └── nginx
│ │ │ └── action.yml
│ │ ├── huggingface
│ │ │ └── huggingface
│ │ │ └── action.yml
│ │ ├── koofr
│ │ │ └── koofr
│ │ │ └── disable_action.yml
│ │ ├── memcached
│ │ │ ├── memcached
│ │ │ │ └── action.yml
│ │ │ └── memcached_with_auth
│ │ │ └── action.yml
│ │ ├── memory
│ │ │ └── memory
│ │ │ └── action.yml
│ │ ├── mini_moka
│ │ │ └── mini_moka
│ │ │ └── action.yml
│ │ ├── moka
│ │ │ └── moka
│ │ │ └── action.yml
│ │ ├── mongodb
│ │ │ ├── mongodb_with_basic_auth
│ │ │ │ └── action.yml
│ │ │ └── mongodb_with_no_auth
│ │ │ └── action.yml
│ │ ├── monoiofs
│ │ │ └── monoiofs
│ │ │ └── action.yml
│ │ ├── mysql
│ │ │ └── mysql
│ │ │ └── action.yml
│ │ ├── oss
│ │ │ ├── oss
│ │ │ │ └── action.yml
│ │ │ └── oss_with_versioning
│ │ │ └── action.yml
│ │ ├── persy
│ │ │ └── persy
│ │ │ └── action.yml
│ │ ├── postgresql
│ │ │ └── postgresql
│ │ │ └── action.yml
│ │ ├── redb
│ │ │ └── redb
│ │ │ └── action.yml
│ │ ├── redis
│ │ │ ├── dragonfly
│ │ │ │ └── action.yml
│ │ │ ├── kvrocks
│ │ │ │ └── action.yml
│ │ │ ├── redis
│ │ │ │ └── action.yml
│ │ │ ├── redis_tls
│ │ │ │ └── action.yml
│ │ │ ├── redis_with_cluster
│ │ │ │ └── action.yml
│ │ │ └── redis_with_cluster_tls
│ │ │ └── action.yml
│ │ ├── rocksdb
│ │ │ └── rocksdb
│ │ │ └── action.yml
│ │ ├── s3
│ │ │ ├── 0_minio_s3
│ │ │ │ └── action.yml
│ │ │ ├── aws_s3
│ │ │ │ └── action.yml
│ │ │ ├── aws_s3_with_list_objects_v1
│ │ │ │ └── action.yml
│ │ │ ├── aws_s3_with_sse_c
│ │ │ │ └── action.yml
│ │ │ ├── aws_s3_with_versioning
│ │ │ │ └── action.yml
│ │ │ ├── aws_s3_with_virtual_host
│ │ │ │ └── action.yml
│ │ │ ├── ceph_radios_s3_with_versioning
│ │ │ │ └── disable_action.yml
│ │ │ ├── ceph_rados_s3
│ │ │ │ └── disable_action.yml
│ │ │ ├── minio_s3_with_anonymous
│ │ │ │ └── action.yml
│ │ │ ├── minio_s3_with_list_objects_v1
│ │ │ │ └── action.yml
│ │ │ ├── minio_s3_with_versioning
│ │ │ │ └── action.yml
│ │ │ └── r2
│ │ │ └── disabled_action.yml
│ │ ├── seafile
│ │ │ └── seafile
│ │ │ └── action.yml
│ │ ├── sftp
│ │ │ ├── sftp
│ │ │ │ └── action.yml
│ │ │ └── sftp_with_default_root
│ │ │ └── action.yml
│ │ ├── sled
│ │ │ ├── sled
│ │ │ │ └── action.yml
│ │ │ └── sled_with_tree
│ │ │ └── action.yml
│ │ ├── sqlite
│ │ │ └── sqlite
│ │ │ └── action.yml
│ │ ├── swift
│ │ │ ├── ceph_rados_swift
│ │ │ │ └── action.yml
│ │ │ └── swift
│ │ │ └── action.yml
│ │ ├── tikv
│ │ │ └── tikv
│ │ │ └── disable_action.yml
│ │ ├── webdav
│ │ │ ├── 0_nginx
│ │ │ │ └── action.yml
│ │ │ ├── jfrog
│ │ │ │ └── disabled_action.yml
│ │ │ ├── nextcloud
│ │ │ │ └── action.yml
│ │ │ ├── nginx_with_empty_password
│ │ │ │ └── action.yml
│ │ │ ├── nginx_with_password
│ │ │ │ └── action.yml
│ │ │ ├── nginx_with_redirect
│ │ │ │ └── action.yml
│ │ │ └── owncloud
│ │ │ └── action.yml
│ │ └── webhdfs
│ │ ├── webhdfs
│ │ │ └── action.yml
│ │ ├── webhdfs_with_list_batch_disabled
│ │ │ └── action.yml
│ │ └── webhdfs_with_user_name
│ │ └── action.yml
│ └── workflows
│ ├── ci_bindings_c.yml
│ ├── ci_bindings_cpp.yml
│ ├── ci_bindings_d.yml
│ ├── ci_bindings_dart.yml
│ ├── ci_bindings_dotnet.yml
│ ├── ci_bindings_go.yml
│ ├── ci_bindings_haskell.yml
│ ├── ci_bindings_java.yml
│ ├── ci_bindings_lua.yml
│ ├── ci_bindings_nodejs.yml
│ ├── ci_bindings_ocaml.yml
│ ├── ci_bindings_php.yml
│ ├── ci_bindings_python.yml
│ ├── ci_bindings_ruby.yml
│ ├── ci_bindings_swift.yml
│ ├── ci_bindings_zig.yml
│ ├── ci_check.yml
│ ├── ci_core.yml
│ ├── ci_integration_dav_server.yml
│ ├── ci_integration_object_store.yml
│ ├── ci_integration_parquet.yml
│ ├── ci_integration_spring.yml
│ ├── ci_integration_unftp_sbe.yml
│ ├── ci_odev.yml
│ ├── ci_weekly_update.yml
│ ├── discussion-thread-link.yml
│ ├── docs.yml
│ ├── full-ci-promote.yml
│ ├── release_dart.yml
│ ├── release_java.yml
│ ├── release_nodejs.yml
│ ├── release_python.yml
│ ├── release_ruby.yml
│ ├── release_rust.yml
│ ├── service_test_ghac.yml
│ ├── test_behavior_binding_c.yml
│ ├── test_behavior_binding_cpp.yml
│ ├── test_behavior_binding_go.yml
│ ├── test_behavior_binding_java.yml
│ ├── test_behavior_binding_nodejs.yml
│ ├── test_behavior_binding_python.yml
│ ├── test_behavior_core.yml
│ ├── test_behavior_integration_object_store.yml
│ ├── test_behavior.yml
│ ├── test_edge.yml
│ └── test_fuzz.yml
├── .gitignore
├── .taplo.toml
├── .typos.toml
├── .vscode
│ └── settings.json
├── .yamlfmt
├── AGENTS.md
├── bindings
│ ├── java
│ │ ├── .cargo
│ │ │ └── config.toml
│ │ ├── .gitignore
│ │ ├── .mvn
│ │ │ └── wrapper
│ │ │ └── maven-wrapper.properties
│ │ ├── Cargo.toml
│ │ ├── DEPENDENCIES.md
│ │ ├── DEPENDENCIES.rust.tsv
│ │ ├── mvnw
│ │ ├── mvnw.cmd
│ │ ├── pom.xml
│ │ ├── README.md
│ │ ├── src
│ │ │ ├── async_operator.rs
│ │ │ ├── convert.rs
│ │ │ ├── error.rs
│ │ │ ├── executor.rs
│ │ │ ├── layer.rs
│ │ │ ├── lib.rs
│ │ │ ├── main
│ │ │ │ ├── java
│ │ │ │ │ └── org
│ │ │ │ │ └── apache
│ │ │ │ │ └── opendal
│ │ │ │ │ ├── AsyncExecutor.java
│ │ │ │ │ ├── AsyncOperator.java
│ │ │ │ │ ├── Capability.java
│ │ │ │ │ ├── Entry.java
│ │ │ │ │ ├── Environment.java
│ │ │ │ │ ├── layer
│ │ │ │ │ │ ├── ConcurrentLimitLayer.java
│ │ │ │ │ │ ├── package-info.java
│ │ │ │ │ │ └── RetryLayer.java
│ │ │ │ │ ├── Layer.java
│ │ │ │ │ ├── ListOptions.java
│ │ │ │ │ ├── Metadata.java
│ │ │ │ │ ├── NativeLibrary.java
│ │ │ │ │ ├── NativeObject.java
│ │ │ │ │ ├── OpenDAL.java
│ │ │ │ │ ├── OpenDALException.java
│ │ │ │ │ ├── Operator.java
│ │ │ │ │ ├── OperatorInfo.java
│ │ │ │ │ ├── OperatorInputStream.java
│ │ │ │ │ ├── OperatorOutputStream.java
│ │ │ │ │ ├── package-info.java
│ │ │ │ │ ├── PresignedRequest.java
│ │ │ │ │ ├── ReadOptions.java
│ │ │ │ │ ├── ServiceConfig.java
│ │ │ │ │ ├── StatOptions.java
│ │ │ │ │ └── WriteOptions.java
│ │ │ │ └── resources
│ │ │ │ ├── bindings.properties
│ │ │ │ └── META-INF
│ │ │ │ └── NOTICE
│ │ │ ├── operator_input_stream.rs
│ │ │ ├── operator_output_stream.rs
│ │ │ ├── operator.rs
│ │ │ ├── test
│ │ │ │ └── java
│ │ │ │ └── org
│ │ │ │ └── apache
│ │ │ │ └── opendal
│ │ │ │ └── test
│ │ │ │ ├── AsyncExecutorTest.java
│ │ │ │ ├── behavior
│ │ │ │ │ ├── AsyncCopyTest.java
│ │ │ │ │ ├── AsyncCreateDirTest.java
│ │ │ │ │ ├── AsyncListTest.java
│ │ │ │ │ ├── AsyncPresignTest.java
│ │ │ │ │ ├── AsyncReadOnlyTest.java
│ │ │ │ │ ├── AsyncRenameTest.java
│ │ │ │ │ ├── AsyncStatOptionsTest.java
│ │ │ │ │ ├── AsyncWriteOptionsTest.java
│ │ │ │ │ ├── AsyncWriteTest.java
│ │ │ │ │ ├── BehaviorExtension.java
│ │ │ │ │ ├── BehaviorTestBase.java
│ │ │ │ │ ├── BlockingCopyTest.java
│ │ │ │ │ ├── BlockingCreateDirTest.java
│ │ │ │ │ ├── BlockingListTest.java
│ │ │ │ │ ├── BlockingReadOnlyTest.java
│ │ │ │ │ ├── BlockingRenameTest.java
│ │ │ │ │ ├── BlockingStatOptionsTest.java
│ │ │ │ │ ├── BlockingWriteOptionTest.java
│ │ │ │ │ ├── BlockingWriteTest.java
│ │ │ │ │ └── RegressionTest.java
│ │ │ │ ├── condition
│ │ │ │ │ └── OpenDALExceptionCondition.java
│ │ │ │ ├── LayerTest.java
│ │ │ │ ├── MetadataTest.java
│ │ │ │ ├── OperatorDuplicateTest.java
│ │ │ │ ├── OperatorInfoTest.java
│ │ │ │ ├── OperatorInputOutputStreamTest.java
│ │ │ │ ├── OperatorUtf8DecodeTest.java
│ │ │ │ └── UtilityTest.java
│ │ │ └── utility.rs
│ │ ├── tools
│ │ │ └── build.py
│ │ ├── upgrade.md
│ │ └── users.md
│ ├── nodejs
│ │ ├── .cargo
│ │ │ └── config.toml
│ │ ├── .gitignore
│ │ ├── .node-version
│ │ ├── .npmignore
│ │ ├── .npmrc
│ │ ├── .prettierignore
│ │ ├── benchmark
│ │ │ ├── deno.ts
│ │ │ ├── node.js
│ │ │ └── README.md
│ │ ├── build.rs
│ │ ├── Cargo.toml
│ │ ├── CONTRIBUTING.md
│ │ ├── DEPENDENCIES.md
│ │ ├── DEPENDENCIES.rust.tsv
│ │ ├── devbox.json
│ │ ├── devbox.lock
│ │ ├── generated.d.ts
│ │ ├── generated.js
│ │ ├── index.cjs
│ │ ├── index.d.ts
│ │ ├── index.mjs
│ │ ├── npm
│ │ │ ├── darwin-arm64
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ ├── darwin-x64
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ ├── linux-arm64-gnu
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ ├── linux-arm64-musl
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ ├── linux-x64-gnu
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ ├── linux-x64-musl
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ ├── win32-arm64-msvc
│ │ │ │ ├── package.json
│ │ │ │ └── README.md
│ │ │ └── win32-x64-msvc
│ │ │ ├── package.json
│ │ │ └── README.md
│ │ ├── package.json
│ │ ├── pnpm-lock.yaml
│ │ ├── README.md
│ │ ├── scripts
│ │ │ └── header.mjs
│ │ ├── src
│ │ │ ├── capability.rs
│ │ │ ├── layer.rs
│ │ │ ├── lib.rs
│ │ │ └── options.rs
│ │ ├── tests
│ │ │ ├── service.test.mjs
│ │ │ ├── suites
│ │ │ │ ├── async.suite.mjs
│ │ │ │ ├── asyncDeleteOptions.suite.mjs
│ │ │ │ ├── asyncLister.suite.mjs
│ │ │ │ ├── asyncListOptions.suite.mjs
│ │ │ │ ├── asyncReadOptions.suite.mjs
│ │ │ │ ├── asyncStatOptions.suite.mjs
│ │ │ │ ├── asyncWriteOptions.suite.mjs
│ │ │ │ ├── index.mjs
│ │ │ │ ├── layer.suite.mjs
│ │ │ │ ├── services.suite.mjs
│ │ │ │ ├── sync.suite.mjs
│ │ │ │ ├── syncDeleteOptions.suite.mjs
│ │ │ │ ├── syncLister.suite.mjs
│ │ │ │ ├── syncListOptions.suite.mjs
│ │ │ │ ├── syncReadOptions.suite.mjs
│ │ │ │ ├── syncStatOptions.suite.mjs
│ │ │ │ └── syncWriteOptions.suite.mjs
│ │ │ └── utils.mjs
│ │ ├── theme
│ │ │ ├── index.tsx
│ │ │ └── package.json
│ │ ├── tsconfig.json
│ │ ├── tsconfig.theme.json
│ │ ├── typedoc.json
│ │ ├── upgrade.md
│ │ └── vitest.config.mjs
│ ├── python
│ │ ├── .gitignore
│ │ ├── benchmark
│ │ │ ├── async_opendal_benchmark.py
│ │ │ ├── async_origin_s3_benchmark_with_gevent.py
│ │ │ └── README.md
│ │ ├── Cargo.toml
│ │ ├── CONTRIBUTING.md
│ │ ├── DEPENDENCIES.md
│ │ ├── DEPENDENCIES.rust.tsv
│ │ ├── docs
│ │ │ ├── api
│ │ │ │ ├── async_file.md
│ │ │ │ ├── async_operator.md
│ │ │ │ ├── capability.md
│ │ │ │ ├── exceptions.md
│ │ │ │ ├── file.md
│ │ │ │ ├── layers.md
│ │ │ │ ├── operator.md
│ │ │ │ └── types.md
│ │ │ └── index.md
│ │ ├── justfile
│ │ ├── mkdocs.yml
│ │ ├── pyproject.toml
│ │ ├── pyrightconfig.json
│ │ ├── python
│ │ │ └── opendal
│ │ │ ├── __init__.py
│ │ │ ├── capability.pyi
│ │ │ ├── exceptions.pyi
│ │ │ ├── file.pyi
│ │ │ ├── layers.pyi
│ │ │ ├── operator.pyi
│ │ │ ├── py.typed
│ │ │ ├── services.pyi
│ │ │ └── types.pyi
│ │ ├── README.md
│ │ ├── ruff.toml
│ │ ├── src
│ │ │ ├── capability.rs
│ │ │ ├── errors.rs
│ │ │ ├── file.rs
│ │ │ ├── layers.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── metadata.rs
│ │ │ ├── operator.rs
│ │ │ ├── options.rs
│ │ │ ├── services.rs
│ │ │ └── utils.rs
│ │ ├── template
│ │ │ └── module.html.jinja2
│ │ ├── tests
│ │ │ ├── conftest.py
│ │ │ ├── test_async_check.py
│ │ │ ├── test_async_copy.py
│ │ │ ├── test_async_delete.py
│ │ │ ├── test_async_exists.py
│ │ │ ├── test_async_list.py
│ │ │ ├── test_async_pickle_types.py
│ │ │ ├── test_async_rename.py
│ │ │ ├── test_capability.py
│ │ │ ├── test_exceptions.py
│ │ │ ├── test_pickle_rw.py
│ │ │ ├── test_read.py
│ │ │ ├── test_sync_check.py
│ │ │ ├── test_sync_copy.py
│ │ │ ├── test_sync_delete.py
│ │ │ ├── test_sync_exists.py
│ │ │ ├── test_sync_list.py
│ │ │ ├── test_sync_pickle_types.py
│ │ │ ├── test_sync_rename.py
│ │ │ └── test_write.py
│ │ ├── upgrade.md
│ │ ├── users.md
│ │ └── uv.lock
│ └── README.md
├── CHANGELOG.md
├── CITATION.cff
├── CLAUDE.md
├── CONTRIBUTING.md
├── core
│ ├── benches
│ │ ├── ops
│ │ │ ├── main.rs
│ │ │ ├── read.rs
│ │ │ ├── README.md
│ │ │ ├── utils.rs
│ │ │ └── write.rs
│ │ ├── README.md
│ │ ├── types
│ │ │ ├── buffer.rs
│ │ │ ├── main.rs
│ │ │ ├── README.md
│ │ │ └── tasks.rs
│ │ ├── vs_fs
│ │ │ ├── Cargo.toml
│ │ │ ├── README.md
│ │ │ └── src
│ │ │ └── main.rs
│ │ └── vs_s3
│ │ ├── Cargo.toml
│ │ ├── README.md
│ │ └── src
│ │ └── main.rs
│ ├── Cargo.lock
│ ├── Cargo.toml
│ ├── CHANGELOG.md
│ ├── CONTRIBUTING.md
│ ├── core
│ │ ├── Cargo.toml
│ │ └── src
│ │ ├── blocking
│ │ │ ├── delete.rs
│ │ │ ├── list.rs
│ │ │ ├── mod.rs
│ │ │ ├── operator.rs
│ │ │ ├── read
│ │ │ │ ├── buffer_iterator.rs
│ │ │ │ ├── mod.rs
│ │ │ │ ├── reader.rs
│ │ │ │ ├── std_bytes_iterator.rs
│ │ │ │ └── std_reader.rs
│ │ │ └── write
│ │ │ ├── mod.rs
│ │ │ ├── std_writer.rs
│ │ │ └── writer.rs
│ │ ├── docs
│ │ │ ├── comparisons
│ │ │ │ ├── mod.rs
│ │ │ │ └── vs_object_store.md
│ │ │ ├── concepts.rs
│ │ │ ├── internals
│ │ │ │ ├── accessor.rs
│ │ │ │ ├── layer.rs
│ │ │ │ └── mod.rs
│ │ │ ├── mod.rs
│ │ │ ├── performance
│ │ │ │ ├── concurrent_write.md
│ │ │ │ ├── http_optimization.md
│ │ │ │ └── mod.rs
│ │ │ ├── rfcs
│ │ │ │ ├── 0000_example.md
│ │ │ │ ├── 0000_foyer_integration.md
│ │ │ │ ├── 0041_object_native_api.md
│ │ │ │ ├── 0044_error_handle.md
│ │ │ │ ├── 0057_auto_region.md
│ │ │ │ ├── 0069_object_stream.md
│ │ │ │ ├── 0090_limited_reader.md
│ │ │ │ ├── 0112_path_normalization.md
│ │ │ │ ├── 0191_async_streaming_io.md
│ │ │ │ ├── 0203_remove_credential.md
│ │ │ │ ├── 0221_create_dir.md
│ │ │ │ ├── 0247_retryable_error.md
│ │ │ │ ├── 0293_object_id.md
│ │ │ │ ├── 0337_dir_entry.md
│ │ │ │ ├── 0409_accessor_capabilities.md
│ │ │ │ ├── 0413_presign.md
│ │ │ │ ├── 0423_command_line_interface.md
│ │ │ │ ├── 0429_init_from_iter.md
│ │ │ │ ├── 0438_multipart.md
│ │ │ │ ├── 0443_gateway.md
│ │ │ │ ├── 0501_new_builder.md
│ │ │ │ ├── 0554_write_refactor.md
│ │ │ │ ├── 0561_list_metadata_reuse.md
│ │ │ │ ├── 0599_blocking_api.md
│ │ │ │ ├── 0623_redis_service.md
│ │ │ │ ├── 0627_split_capabilities.md
│ │ │ │ ├── 0661_path_in_accessor.md
│ │ │ │ ├── 0793_generic_kv_services.md
│ │ │ │ ├── 0926_object_reader.md
│ │ │ │ ├── 0977_refactor_error.md
│ │ │ │ ├── 1085_object_handler.md
│ │ │ │ ├── 1391_object_metadataer.md
│ │ │ │ ├── 1398_query_based_metadata.md
│ │ │ │ ├── 1420_object_writer.md
│ │ │ │ ├── 1477_remove_object_concept.md
│ │ │ │ ├── 1735_operation_extension.md
│ │ │ │ ├── 2083_writer_sink_api.md
│ │ │ │ ├── 2133_append_api.md
│ │ │ │ ├── 2299_chain_based_operator_api.md
│ │ │ │ ├── 2602_object_versioning.md
│ │ │ │ ├── 2758_merge_append_into_write.md
│ │ │ │ ├── 2774_lister_api.md
│ │ │ │ ├── 2779_list_with_metakey.md
│ │ │ │ ├── 2852_native_capability.md
│ │ │ │ ├── 2884_merge_range_read_into_read.md
│ │ │ │ ├── 3017_remove_write_copy_from.md
│ │ │ │ ├── 3197_config.md
│ │ │ │ ├── 3232_align_list_api.md
│ │ │ │ ├── 3243_list_prefix.md
│ │ │ │ ├── 3356_lazy_reader.md
│ │ │ │ ├── 3526_list_recursive.md
│ │ │ │ ├── 3574_concurrent_stat_in_list.md
│ │ │ │ ├── 3734_buffered_reader.md
│ │ │ │ ├── 3898_concurrent_writer.md
│ │ │ │ ├── 3911_deleter_api.md
│ │ │ │ ├── 4382_range_based_read.md
│ │ │ │ ├── 4638_executor.md
│ │ │ │ ├── 5314_remove_metakey.md
│ │ │ │ ├── 5444_operator_from_uri.md
│ │ │ │ ├── 5479_context.md
│ │ │ │ ├── 5485_conditional_reader.md
│ │ │ │ ├── 5495_list_with_deleted.md
│ │ │ │ ├── 5556_write_returns_metadata.md
│ │ │ │ ├── 5871_read_returns_metadata.md
│ │ │ │ ├── 6189_remove_native_blocking.md
│ │ │ │ ├── 6209_glob_support.md
│ │ │ │ ├── 6213_options_api.md
│ │ │ │ ├── 6678_simulate_layer.md
│ │ │ │ ├── 6707_capability_override_layer.md
│ │ │ │ ├── 6817_checksum.md
│ │ │ │ ├── 6828_core.md
│ │ │ │ ├── 7130_route_layer.md
│ │ │ │ ├── mod.rs
│ │ │ │ └── README.md
│ │ │ └── upgrade.md
│ │ ├── layers
│ │ │ ├── complete.rs
│ │ │ ├── correctness_check.rs
│ │ │ ├── error_context.rs
│ │ │ ├── http_client.rs
│ │ │ ├── mod.rs
│ │ │ ├── simulate.rs
│ │ │ └── type_eraser.rs
│ │ ├── lib.rs
│ │ ├── raw
│ │ │ ├── accessor.rs
│ │ │ ├── atomic_util.rs
│ │ │ ├── enum_utils.rs
│ │ │ ├── futures_util.rs
│ │ │ ├── http_util
│ │ │ │ ├── body.rs
│ │ │ │ ├── bytes_content_range.rs
│ │ │ │ ├── bytes_range.rs
│ │ │ │ ├── client.rs
│ │ │ │ ├── error.rs
│ │ │ │ ├── header.rs
│ │ │ │ ├── mod.rs
│ │ │ │ ├── multipart.rs
│ │ │ │ └── uri.rs
│ │ │ ├── layer.rs
│ │ │ ├── mod.rs
│ │ │ ├── oio
│ │ │ │ ├── buf
│ │ │ │ │ ├── flex_buf.rs
│ │ │ │ │ ├── mod.rs
│ │ │ │ │ ├── pooled_buf.rs
│ │ │ │ │ └── queue_buf.rs
│ │ │ │ ├── delete
│ │ │ │ │ ├── api.rs
│ │ │ │ │ ├── batch_delete.rs
│ │ │ │ │ ├── mod.rs
│ │ │ │ │ └── one_shot_delete.rs
│ │ │ │ ├── entry.rs
│ │ │ │ ├── list
│ │ │ │ │ ├── api.rs
│ │ │ │ │ ├── flat_list.rs
│ │ │ │ │ ├── hierarchy_list.rs
│ │ │ │ │ ├── mod.rs
│ │ │ │ │ ├── page_list.rs
│ │ │ │ │ └── prefix_list.rs
│ │ │ │ ├── mod.rs
│ │ │ │ ├── read
│ │ │ │ │ ├── api.rs
│ │ │ │ │ └── mod.rs
│ │ │ │ └── write
│ │ │ │ ├── api.rs
│ │ │ │ ├── append_write.rs
│ │ │ │ ├── block_write.rs
│ │ │ │ ├── mod.rs
│ │ │ │ ├── multipart_write.rs
│ │ │ │ ├── one_shot_write.rs
│ │ │ │ └── position_write.rs
│ │ │ ├── operation.rs
│ │ │ ├── ops.rs
│ │ │ ├── path_cache.rs
│ │ │ ├── path.rs
│ │ │ ├── rps.rs
│ │ │ ├── serde_util.rs
│ │ │ ├── std_io_util.rs
│ │ │ ├── time.rs
│ │ │ ├── tokio_util.rs
│ │ │ └── version.rs
│ │ ├── services
│ │ │ ├── memory
│ │ │ │ ├── backend.rs
│ │ │ │ ├── config.rs
│ │ │ │ ├── core.rs
│ │ │ │ ├── deleter.rs
│ │ │ │ ├── docs.md
│ │ │ │ ├── lister.rs
│ │ │ │ ├── mod.rs
│ │ │ │ └── writer.rs
│ │ │ └── mod.rs
│ │ └── types
│ │ ├── buffer.rs
│ │ ├── builder.rs
│ │ ├── capability.rs
│ │ ├── context
│ │ │ ├── mod.rs
│ │ │ ├── read.rs
│ │ │ └── write.rs
│ │ ├── delete
│ │ │ ├── deleter.rs
│ │ │ ├── futures_delete_sink.rs
│ │ │ ├── input.rs
│ │ │ └── mod.rs
│ │ ├── entry.rs
│ │ ├── error.rs
│ │ ├── execute
│ │ │ ├── api.rs
│ │ │ ├── executor.rs
│ │ │ ├── executors
│ │ │ │ ├── mod.rs
│ │ │ │ └── tokio_executor.rs
│ │ │ └── mod.rs
│ │ ├── list.rs
│ │ ├── metadata.rs
│ │ ├── mod.rs
│ │ ├── mode.rs
│ │ ├── operator
│ │ │ ├── builder.rs
│ │ │ ├── info.rs
│ │ │ ├── mod.rs
│ │ │ ├── operator_futures.rs
│ │ │ ├── operator.rs
│ │ │ ├── registry.rs
│ │ │ └── uri.rs
│ │ ├── options.rs
│ │ ├── read
│ │ │ ├── buffer_stream.rs
│ │ │ ├── futures_async_reader.rs
│ │ │ ├── futures_bytes_stream.rs
│ │ │ ├── mod.rs
│ │ │ └── reader.rs
│ │ └── write
│ │ ├── buffer_sink.rs
│ │ ├── futures_async_writer.rs
│ │ ├── futures_bytes_sink.rs
│ │ ├── mod.rs
│ │ └── writer.rs
│ ├── DEPENDENCIES.md
│ ├── DEPENDENCIES.rust.tsv
│ ├── edge
│ │ ├── file_write_on_full_disk
│ │ │ ├── Cargo.toml
│ │ │ ├── README.md
│ │ │ └── src
│ │ │ └── main.rs
│ │ ├── README.md
│ │ ├── s3_aws_assume_role_with_web_identity
│ │ │ ├── Cargo.toml
│ │ │ ├── README.md
│ │ │ └── src
│ │ │ └── main.rs
│ │ └── s3_read_on_wasm
│ │ ├── .gitignore
│ │ ├── Cargo.toml
│ │ ├── README.md
│ │ ├── src
│ │ │ └── lib.rs
│ │ └── webdriver.json
│ ├── fuzz
│ │ ├── .gitignore
│ │ ├── Cargo.toml
│ │ ├── fuzz_reader.rs
│ │ ├── fuzz_writer.rs
│ │ └── README.md
│ ├── layers
│ │ ├── async-backtrace
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── await-tree
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── capability-check
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── chaos
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── concurrent-limit
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── dtrace
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── fastmetrics
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── fastrace
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── hotpath
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── immutable-index
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── logging
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── metrics
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── mime-guess
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── observe-metrics-common
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── otelmetrics
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── oteltrace
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── prometheus
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── prometheus-client
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── retry
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── tail-cut
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── throttle
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── timeout
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ └── tracing
│ │ ├── Cargo.toml
│ │ └── src
│ │ └── lib.rs
│ ├── LICENSE
│ ├── README.md
│ ├── services
│ │ ├── aliyun-drive
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── alluxio
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── azblob
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── azdls
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── azfile
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── azure-common
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ └── lib.rs
│ │ ├── b2
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── cacache
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── cloudflare-kv
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── model.rs
│ │ │ └── writer.rs
│ │ ├── compfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── reader.rs
│ │ │ └── writer.rs
│ │ ├── cos
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── d1
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── model.rs
│ │ │ └── writer.rs
│ │ ├── dashmap
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── dbfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── dropbox
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── builder.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── etcd
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── foundationdb
│ │ │ ├── build.rs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── fs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── reader.rs
│ │ │ └── writer.rs
│ │ ├── ftp
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── err.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── reader.rs
│ │ │ └── writer.rs
│ │ ├── gcs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── uri.rs
│ │ │ └── writer.rs
│ │ ├── gdrive
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── builder.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── ghac
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── github
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── mod.rs
│ │ │ └── writer.rs
│ │ ├── gridfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── hdfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── reader.rs
│ │ │ └── writer.rs
│ │ ├── hdfs-native
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── reader.rs
│ │ │ └── writer.rs
│ │ ├── http
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ └── lib.rs
│ │ ├── huggingface
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ └── lister.rs
│ │ ├── ipfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── ipld.rs
│ │ │ └── lib.rs
│ │ ├── ipmfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── builder.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── koofr
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── lakefs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── memcached
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── binary.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── mini_moka
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── moka
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── mongodb
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── monoiofs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── reader.rs
│ │ │ └── writer.rs
│ │ ├── mysql
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── obs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── onedrive
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── builder.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── graph_model.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── opfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ └── utils.rs
│ │ ├── oss
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── pcloud
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── persy
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── postgresql
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── redb
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── redis
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── delete.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── rocksdb
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── s3
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── compatible_services.md
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── mod.rs
│ │ │ └── writer.rs
│ │ ├── seafile
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── sftp
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── reader.rs
│ │ │ ├── utils.rs
│ │ │ └── writer.rs
│ │ ├── sled
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── sqlite
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── surrealdb
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── swift
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── compatible_services.md
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── tikv
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── upyun
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── vercel-artifacts
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── builder.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ └── writer.rs
│ │ ├── vercel-blob
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── webdav
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ └── writer.rs
│ │ ├── webhdfs
│ │ │ ├── Cargo.toml
│ │ │ └── src
│ │ │ ├── backend.rs
│ │ │ ├── config.rs
│ │ │ ├── core.rs
│ │ │ ├── deleter.rs
│ │ │ ├── docs.md
│ │ │ ├── error.rs
│ │ │ ├── lib.rs
│ │ │ ├── lister.rs
│ │ │ ├── message.rs
│ │ │ └── writer.rs
│ │ └── yandex-disk
│ │ ├── Cargo.toml
│ │ └── src
│ │ ├── backend.rs
│ │ ├── config.rs
│ │ ├── core.rs
│ │ ├── deleter.rs
│ │ ├── docs.md
│ │ ├── error.rs
│ │ ├── lib.rs
│ │ ├── lister.rs
│ │ └── writer.rs
│ ├── src
│ │ └── lib.rs
│ ├── testkit
│ │ ├── Cargo.toml
│ │ └── src
│ │ ├── lib.rs
│ │ ├── read.rs
│ │ ├── utils.rs
│ │ └── write.rs
│ ├── tests
│ │ ├── behavior
│ │ │ ├── async_copy.rs
│ │ │ ├── async_create_dir.rs
│ │ │ ├── async_delete.rs
│ │ │ ├── async_list.rs
│ │ │ ├── async_presign.rs
│ │ │ ├── async_read.rs
│ │ │ ├── async_rename.rs
│ │ │ ├── async_stat.rs
│ │ │ ├── async_write.rs
│ │ │ ├── main.rs
│ │ │ ├── README.md
│ │ │ └── utils.rs
│ │ └── data
│ │ ├── normal_dir
│ │ │ └── .gitkeep
│ │ ├── normal_file.txt
│ │ ├── special_dir !@#$%^&()_+-=;',
│ │ │ └── .gitkeep
│ │ └── special_file !@#$%^&()_+-=;',.txt
│ ├── upgrade.md
│ └── users.md
├── deny.toml
├── DEPENDENCIES.md
├── dev
│ ├── Cargo.lock
│ ├── Cargo.toml
│ ├── README.md
│ └── src
│ ├── generate
│ │ ├── java.j2
│ │ ├── java.rs
│ │ ├── mod.rs
│ │ ├── parser.rs
│ │ ├── python.j2
│ │ └── python.rs
│ ├── main.rs
│ └── release
│ ├── mod.rs
│ └── package.rs
├── doap.rdf
├── fixtures
│ ├── alluxio
│ │ └── docker-compose-alluxio.yml
│ ├── azblob
│ │ └── docker-compose-azurite.yml
│ ├── data
│ │ ├── normal_dir
│ │ │ └── .gitkeep
│ │ ├── normal_file.txt
│ │ ├── special_dir !@#$%^&()_+-=;',
│ │ │ └── .gitkeep
│ │ └── special_file !@#$%^&()_+-=;',.txt
│ ├── etcd
│ │ ├── ca-key.pem
│ │ ├── ca.pem
│ │ ├── client-key.pem
│ │ ├── client.pem
│ │ ├── docker-compose-cluster.yml
│ │ ├── docker-compose-standalone-tls.yml
│ │ ├── docker-compose-standalone.yml
│ │ ├── server-key.pem
│ │ └── server.pem
│ ├── ftp
│ │ └── docker-compose-vsftpd.yml
│ ├── hdfs
│ │ ├── azurite-azblob-core-site.xml
│ │ ├── docker-compose-hdfs-cluster.yml
│ │ ├── gcs-core-site.xml
│ │ ├── hdfs-site.xml
│ │ └── minio-s3-core-site.xml
│ ├── http
│ │ ├── Caddyfile
│ │ ├── docker-compose-caddy.yml
│ │ ├── docker-compose-nginx.yml
│ │ └── nginx.conf
│ ├── libsql
│ │ ├── docker-compose-auth.yml
│ │ └── docker-compose.yml
│ ├── memcached
│ │ ├── docker-compose-memcached-with-auth.yml
│ │ └── docker-compose-memcached.yml
│ ├── mongodb
│ │ ├── docker-compose-basic-auth.yml
│ │ └── docker-compose-no-auth.yml
│ ├── mysql
│ │ ├── docker-compose.yml
│ │ └── init.sql
│ ├── postgresql
│ │ ├── docker-compose.yml
│ │ └── init.sql
│ ├── redis
│ │ ├── docker-compose-dragonfly.yml
│ │ ├── docker-compose-kvrocks.yml
│ │ ├── docker-compose-redis-cluster-tls.yml
│ │ ├── docker-compose-redis-cluster.yml
│ │ ├── docker-compose-redis-tls.yml
│ │ ├── docker-compose-redis.yml
│ │ └── ssl
│ │ ├── .gitignore
│ │ ├── ca.crt
│ │ ├── ca.key
│ │ ├── ca.srl
│ │ ├── README.md
│ │ ├── redis.crt
│ │ ├── redis.key
│ │ └── req.conf
│ ├── s3
│ │ ├── docker-compose-ceph-rados.yml
│ │ └── docker-compose-minio.yml
│ ├── seafile
│ │ └── docker-compose-seafile.yml
│ ├── sftp
│ │ ├── change_root_dir.sh
│ │ ├── docker-compose-sftp-with-default-root.yml
│ │ ├── docker-compose-sftp.yml
│ │ ├── health-check.sh
│ │ ├── test_ssh_key
│ │ └── test_ssh_key.pub
│ ├── sqlite
│ │ └── data.sql
│ ├── swift
│ │ ├── docker-compose-ceph-rados.yml
│ │ └── docker-compose-swift.yml
│ ├── tikv
│ │ ├── gen_cert.sh
│ │ ├── pd-tls.toml
│ │ ├── pd.toml
│ │ ├── ssl
│ │ │ ├── ca-key.pem
│ │ │ ├── ca.pem
│ │ │ ├── client-key.pem
│ │ │ ├── client.pem
│ │ │ ├── pd-server-key.pem
│ │ │ ├── pd-server.pem
│ │ │ ├── tikv-server-key.pem
│ │ │ └── tikv-server.pem
│ │ ├── tikv-tls.toml
│ │ └── tikv.toml
│ ├── webdav
│ │ ├── config
│ │ │ └── nginx
│ │ │ └── http.conf
│ │ ├── docker-compose-webdav-jfrog.yml
│ │ ├── docker-compose-webdav-nextcloud.yml
│ │ ├── docker-compose-webdav-owncloud.yml
│ │ ├── docker-compose-webdav-with-auth.yml
│ │ ├── docker-compose-webdav-with-empty-passwd.yml
│ │ ├── docker-compose-webdav.yml
│ │ └── health-check-nextcloud.sh
│ └── webhdfs
│ └── docker-compose-webhdfs.yml
├── justfile
├── LICENSE
├── licenserc.toml
├── NOTICE
├── README.md
├── rust-toolchain.toml
├── rustfmt.toml
└── scripts
├── constants.py
├── dependencies.py
├── merge_local_staging.py
├── README.md
├── verify.py
└── workspace.py
```
# Files
--------------------------------------------------------------------------------
/core/tests/data/normal_dir/.gitkeep:
--------------------------------------------------------------------------------
```
```
--------------------------------------------------------------------------------
/core/tests/data/special_dir !@#$%^&()_+-=;',/.gitkeep:
--------------------------------------------------------------------------------
```
```
--------------------------------------------------------------------------------
/fixtures/data/normal_dir/.gitkeep:
--------------------------------------------------------------------------------
```
```
--------------------------------------------------------------------------------
/fixtures/data/special_dir !@#$%^&()_+-=;',/.gitkeep:
--------------------------------------------------------------------------------
```
```
--------------------------------------------------------------------------------
/bindings/nodejs/.node-version:
--------------------------------------------------------------------------------
```
18
```
--------------------------------------------------------------------------------
/.github/scripts/weekly_update/.python-version:
--------------------------------------------------------------------------------
```
3.12
```
--------------------------------------------------------------------------------
/fixtures/redis/ssl/.gitignore:
--------------------------------------------------------------------------------
```
redis.csr
```
--------------------------------------------------------------------------------
/bindings/nodejs/.npmrc:
--------------------------------------------------------------------------------
```
shell-emulator=true
```
--------------------------------------------------------------------------------
/core/edge/s3_read_on_wasm/.gitignore:
--------------------------------------------------------------------------------
```
pkg/
node_modules/
dist/
```
--------------------------------------------------------------------------------
/core/fuzz/.gitignore:
--------------------------------------------------------------------------------
```
target
corpus
artifacts
coverage
tmp
```
--------------------------------------------------------------------------------
/bindings/java/.gitignore:
--------------------------------------------------------------------------------
```
bin/
.mvn/wrapper/maven-wrapper.jar
Cargo.lock
.project
.factorypath
*.log
.settings/
.classpath
```
--------------------------------------------------------------------------------
/bindings/nodejs/.prettierignore:
--------------------------------------------------------------------------------
```
target
generated.js
generated.d.ts
.yarn
pnpm-lock.yaml
.devbox
devbox.json
devbox.lock
theme/dist/index.js
```
--------------------------------------------------------------------------------
/.github/scripts/weekly_update/.gitignore:
--------------------------------------------------------------------------------
```
# Python-generated files
__pycache__/
*.py[oc]
build/
dist/
wheels/
*.egg-info
.ruff_cache/
# Virtual environments
.venv
```
--------------------------------------------------------------------------------
/bindings/nodejs/.npmignore:
--------------------------------------------------------------------------------
```
.npmrc
target
Cargo.lock
.cargo
.github
npm
.eslintrc
.prettierignore
rustfmt.toml
yarn.lock
pnpm-lock.yaml
*.node
.yarn
__test__
renovate.json
```
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
```
# IDE and editor
.idea
.vscode
!.vscode/settings.json
**/target
**/vendor
dist/
# env files for backends
.env
# profiling
flamegraph.svg
perf.*
**/.DS_Store
# Yarn Integrity file
.yarn-integrity
# Python cache files
**/__pycache__/
```
--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
```
website export-ignore
examples export-ignore
# We will not export unreleased components.
# If any of these are released, remove the line.
bin export-ignore
integrations export-ignore
bindings/c export-ignore
bindings/cpp export-ignore
bindings/d export-ignore
bindings/dart export-ignore
bindings/dotnet export-ignore
bindings/go export-ignore
bindings/haskell export-ignore
bindings/lua export-ignore
bindings/ocaml export-ignore
bindings/php export-ignore
bindings/ruby export-ignore
bindings/swift export-ignore
bindings/zig export-ignore
```
--------------------------------------------------------------------------------
/bindings/python/.gitignore:
--------------------------------------------------------------------------------
```
/target
# Byte-compiled / optimized / DLL files
__pycache__/
.pytest_cache/
*.py[cod]
# C extensions
*.so
# Distribution / packaging
.Python
.venv/
env/
bin/
build/
develop-eggs/
dist/
eggs/
lib/
lib64/
parts/
sdist/
var/
include/
man/
venv/
*.egg-info/
.installed.cfg
*.egg
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
pip-selfcheck.json
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.cache
nosetests.xml
coverage.xml
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
# Rope
.ropeproject
# Django stuff:
*.log
*.pot
.DS_Store
# Sphinx documentation
build/docs
# PyCharm
.idea/
# VSCode
.vscode/
# Pyenv
.python-version
# Generated docs
site/
Cargo.lock
```
--------------------------------------------------------------------------------
/.yamlfmt:
--------------------------------------------------------------------------------
```
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Configuration reference: https://github.com/google/yamlfmt/blob/main/docs/config-file.md
formatter:
retain_line_breaks: true
scan_folded_as_literal: true
```
--------------------------------------------------------------------------------
/.typos.toml:
--------------------------------------------------------------------------------
```toml
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[default.extend-words]
# Random strings.
"Dum" = "Dum"
"Hel" = "Hel"
"ba" = "ba"
"hellow" = "hellow"
# Showed up in examples.
"thw" = "thw"
# Showed up in test.
"hsa" = "hsa"
# Tech words
"WRONLY" = "WRONLY"
"typ" = "typ"
[files]
extend-exclude = [
# Generated SSH Keys.
"fixtures/sftp/test_ssh_key",
"fixtures/sftp/test_ssh_key.pub",
# Generated pnpm locks.
"website/pnpm-lock.yaml",
"CHANGELOG.md",
# dscanner config
"bindings/d/dscanner.ini",
]
```
--------------------------------------------------------------------------------
/.editorconfig:
--------------------------------------------------------------------------------
```
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# EditorConfig helps developers define and maintain consistent
# coding styles between different editors and IDEs
# editorconfig.org
root = true
[*]
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
indent_style = space
indent_size = 4
[*.css]
indent_size = 2
[*.go]
indent_style = tab
[*.html]
indent_size = 2
[*.json]
indent_size = 2
[*.md]
# double whitespace at end of line
# denotes a line break in Markdown
trim_trailing_whitespace = false
indent_size = 2
[*.{mjs,js,jsx}]
indent_size = 2
[*.rdf]
indent_size = 2
[*.toml]
indent_size = 2
[*.{ts,tsx}]
indent_size = 2
[*.{yaml,yml}]
indent_size = 2
[*.rb]
indent_size = 2
```
--------------------------------------------------------------------------------
/.asf.yaml:
--------------------------------------------------------------------------------
```yaml
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# All configurations could be found here: https://github.com/apache/infrastructure-asfyaml/
github:
description: "Apache OpenDAL: One Layer, All Storage."
homepage: https://opendal.apache.org
labels:
- rust
- storage
- s3
- gcs
- azblob
- redis
- hacktoberfest
enabled_merge_buttons:
squash: true
merge: false
rebase: false
pull_requests:
allow_auto_merge: true
allow_update_branch: true
del_branch_on_merge: true
protected_branches:
main:
required_pull_request_reviews:
required_approving_review_count: 1
custom_subjects:
new_discussion: "{title}"
edit_discussion: "Re: {title}"
close_discussion: "Re: {title}"
close_discussion_with_comment: "Re: {title}"
reopen_discussion: "Re: {title}"
new_comment_discussion: "Re: {title}"
edit_comment_discussion: "Re: {title}"
delete_comment_discussion: "Re: {title}"
notifications:
commits: [email protected]
issues: [email protected]
pullrequests: [email protected]
discussions: [email protected]
staging:
profile: ~
autostage: site/*
publish:
whoami: gh-pages
```
--------------------------------------------------------------------------------
/.taplo.toml:
--------------------------------------------------------------------------------
```toml
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
include = ["Cargo.toml", "**/*.toml"]
[formatting]
# Align consecutive entries vertically.
align_entries = false
# Append trailing commas for multi-line arrays.
array_trailing_comma = true
# Expand arrays to multiple lines that exceed the maximum column width.
array_auto_expand = true
# Collapse arrays that don't exceed the maximum column width and don't contain comments.
array_auto_collapse = true
# Omit white space padding from single-line arrays
compact_arrays = true
# Omit white space padding from the start and end of inline tables.
compact_inline_tables = false
# Maximum column width in characters, affects array expansion and collapse, this doesn't take whitespace into account.
# Note that this is not set in stone, and works on a best-effort basis.
column_width = 80
# Indent based on tables and arrays of tables and their subtables, subtables out of order are not indented.
indent_tables = false
# The substring that is used for indentation, should be tabs or spaces (but technically can be anything).
indent_string = ' '
# Add trailing newline at the end of the file if not present.
trailing_newline = true
# Alphabetically reorder keys that are not separated by empty lines.
reorder_keys = true
# Maximum amount of allowed consecutive blank lines. This does not affect the whitespace at the end of the document, as it is always stripped.
allowed_blank_lines = 1
# Use CRLF for line endings.
crlf = false
```
--------------------------------------------------------------------------------
/bindings/nodejs/.gitignore:
--------------------------------------------------------------------------------
```
# Created by https://www.toptal.com/developers/gitignore/api/node
# Edit at https://www.toptal.com/developers/gitignore?templates=node
### Node ###
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
deno.lock
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# TypeScript v1 declaration files
typings/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
.env.test
# parcel-bundler cache (https://parceljs.org/)
.cache
# Next.js build output
.next
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# End of https://www.toptal.com/developers/gitignore/api/node
# Created by https://www.toptal.com/developers/gitignore/api/macos
# Edit at https://www.toptal.com/developers/gitignore?templates=macos
### macOS ###
# General
.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
### macOS Patch ###
# iCloud generated files
*.icloud
# End of https://www.toptal.com/developers/gitignore/api/macos
# Created by https://www.toptal.com/developers/gitignore/api/windows
# Edit at https://www.toptal.com/developers/gitignore?templates=windows
### Windows ###
# Windows thumbnail cache files
Thumbs.db
Thumbs.db:encryptable
ehthumbs.db
ehthumbs_vista.db
# Dump file
*.stackdump
# Folder config file
[Dd]esktop.ini
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Windows Installer files
*.cab
*.msi
*.msix
*.msm
*.msp
# Windows shortcuts
*.lnk
# End of https://www.toptal.com/developers/gitignore/api/windows
#Added by cargo
/target
Cargo.lock
.pnp.*
.yarn
*.node
docs/
```
--------------------------------------------------------------------------------
/.env.example:
--------------------------------------------------------------------------------
```
# memory
# fs
OPENDAL_FS_ROOT=/path/to/dir
OPENDAL_FS_ATOMIC_WRITE_DIR=/path/to/tempdir
# cos
OPENDAL_COS_BUCKET=opendal-testing-1318209832
OPENDAL_COS_ENDPOINT=https://cos.ap-singapore.myqcloud.com
OPENDAL_COS_SECRET_ID=<secret_id>
OPENDAL_COS_SECRET_KEY=<secret_key>
# s3
OPENDAL_S3_BUCKET=<bucket>
OPENDAL_S3_ENDPOINT=<endpoint>
OPENDAL_S3_REGION=<region>
OPENDAL_S3_ACCESS_KEY_ID=<access_key_id>
OPENDAL_S3_SECRET_ACCESS_KEY=<secret_access_key>
# azblob
OPENDAL_AZBLOB_ROOT=/path/to/dir
OPENDAL_AZBLOB_CONTAINER=<container>
OPENDAL_AZBLOB_ENDPOINT=<endpoint>
OPENDAL_AZBLOB_ACCOUNT_NAME=<account_name>
OPENDAL_AZBLOB_ACCOUNT_KEY=<account_key>
# hdfs
OPENDAL_HDFS_ROOT=/path/to/dir
OPENDAL_HDFS_NAME_NODE=<name_node>
OPENDAL_HDFS_ENABLE_APPEND=true
# hdfs-native
OPENDAL_HDFS_NATIVE_ROOT=/path/to/dir
OPENDAL_HDFS_NATIVE_NAME_NODE=<name_node>
OPENDAL_HDFS_NATIVE_ENABLE_APPEND=true
# gcs
OPENDAL_GCS_ROOT=/path/to/dir
OPENDAL_GCS_BUCKET=<bucket>
OPENDAL_GCS_CREDENTIAL=<base64_content>
# obs
OPENDAL_OBS_BUCKET=<bucket>
OPENDAL_OBS_ENDPOINT=<endpoint>
OPENDAL_OBS_ACCESS_KEY_ID=<access_key_id>
OPENDAL_OBS_SECRET_ACCESS_KEY=<secret_access_key>
# oss
OPENDAL_OSS_BUCKET=<bucket>
OPENDAL_OSS_ENDPOINT=<endpoint>
OPENDAL_OSS_ACCESS_KEY_ID=<access_key_id>
OPENDAL_OSS_ACCESS_KEY_SECRET=<access_key_secret>
# ipmfs
OPENDAL_IPMFS_ROOT=/path/to/dir
OPENDAL_IPMFS_ENDPOINT=http://localhost:5001
# ftp
OPENDAL_FTP_ENDPOINT=ftp://<endpoint>
OPENDAL_FTP_ROOT=/path/to/dir
OPENDAL_FTP_USER=<user>
OPENDAL_FTP_PASSWORD=<password>
# ipfs
OPENDAL_IPFS_ROOT=/ipfs/Qmxxxxxxxx
OPENDAL_IPFS_ENDPOINT=http://localhost:8080
# redis
OPENDAL_REDIS_ENDPOINT=tcp://127.0.0.1:6379
# OPENDAL_REDIS_CLUSTER_ENDPOINTS=rediss://127.0.0.1:6380,rediss://127.0.0.1:6381,rediss://127.0.0.1:6382,rediss://127.0.0.1:6383,rediss://127.0.0.1:6384,rediss://127.0.0.1:6385
OPENDAL_REDIS_ROOT=/
OPENDAL_REDIS_DB=0
# rocksdb
OPENDAL_ROCKSDB_DATADIR=/path/to/database
OPENDAL_ROCKSDB_ROOT=/path/to/root
# sftp
OPENDAL_SFTP_ENDPOINT=ssh://<endpoint>
OPENDAL_SFTP_ROOT=/path/to/dir
OPENDAL_SFTP_USER=<user>
OPENDAL_SFTP_KEY=<key_path>
OPENDAL_SFTP_KNOWN_HOSTS_STRATEGY=<accept|add|strict>
# sled
OPENDAL_SLED_DATADIR=/path/to/database
OPENDAL_SLED_TREE=sled-tree
# mini-moka
# moka
# ghac
# memcached
OPENDAL_MEMCACHED_ENDPOINT=tcp://127.0.0.1:11211
OPENDAL_MEMCACHED_ROOT=/
# webdav
OPENDAL_WEBDAV_ROOT=/tmp/opendal/
OPENDAL_WEBDAV_ENDPOINT=http://127.0.0.1:8080
# webhfds
OPENDAL_WEBHDFS_ROOT=/tmp/opendal/
OPENDAL_WEBHDFS_ENDPOINT=http://127.0.0.1:9870
OPENDAL_WEBHDFS_DELEGATION=<delegation>
OPENDAL_WEBHDFS_ATOMIC_WRITE_DIR=.opendal_tmp/
OPENDAL_WEBHDFS_DISABLE_LIST_BATCH=false
# vercel artifacts
OPENDAL_VERCEL_ARTIFACTS_ACCESS_TOKEN=<token>
# onedrive
OPENDAL_ONEDRIVE_ACCESS_TOKEN=<access_token>
# foundationdb
OPENDAL_FOUNDATIONDB_ROOT=/path/to/dir
OPENDAL_FOUNDATIONDB_CONFIG_PATH=/tmp/opendal/foundationdb.conf
# redb
OPENDAL_REDB_DATADIR=/tmp/redb
OPENDAL_REDB_TABLE=redb-table
# cacache
OPENDAL_CACACHE_DATADIR=/tmp/opendal/cacache/
# persy
OPENDAL_PERSY_DATAFILE=/tmp/opendal/test.persy
OPENDAL_PERSY_SEGMENT=data
OPENDAL_PERSY_INDEX=index
#dropbox
OPENDAL_DROPBOX_ROOT=/tmp/opendal/
OPENDAL_DROPBOX_ACCESS_TOKEN=<access_token>
OPENDAL_DROPBOX_REFRESH_TOKEN=<refresh_token>
OPENDAL_DROPBOX_CLIENT_ID=<client_id>
OPENDAL_DROPBOX_CLIENT_SECRET=<client_secret>
# etcd
OPENDAL_ETCD_ENDPOINTS=127.0.0.1:2379
OPENDAL_ETCD_ROOT=/tmp/opendal/
OPENDAL_ETCD_USERNAME=<username>
OPENDAL_ETCD_PASSWORD=<password>
OPENDAL_ETCD_CA_PATH=<ca_path>
OPENDAL_ETCD_CERT_PATH=<cert_path>
OPENDAL_ETCD_KEY_PATH=<key_path>
# google drive
OPENDAL_GDRIVE_ROOT=/tmp/opendal/
OPENDAL_GDRIVE_ACCESS_TOKEN=<access_token>
OPENDAL_GDRIVE_REFRESH_TOKEN=<refresh_token>
OPENDAL_GDRIVE_CLIENT_ID=<client_id>
OPENDAL_GDRIVE_CLIENT_SECRET=<client_secret>
# sqlite
OPENDAL_SQLITE_CONNECTION_STRING=file:///tmp/opendal/test.db
OPENDAL_SQLITE_TABLE=data
OPENDAL_SQLITE_KEY_FIELD=key
OPENDAL_SQLITE_VALUE_FIELD=data
# d1
OPENDAL_D1_TOKEN=<token>
OPENDAL_D1_ACCOUNT_ID=<account_id>
OPENDAL_D1_DATABASE_ID=<database_id>
OPENDAL_D1_TABLE=<table>
OPENDAL_D1_KEY_FIELD=<key_field>
OPENDAL_D1_VALUE_FIELD=<value_field>
# azfile
OPENDAL_AZFILE_ENDPOINT=<endpoint>
OPENDAL_AZFILE_ROOT=/tmp/opendal/
OPENDAL_AZFILE_ACCOUNT_NAME=<account_name>
OPENDAL_AZFILE_ACCOUNT_KEY=<account_key>
OPENDAL_AZFILE_SHARE_NAME=<shared_name>
# openstack swift
OPENDAL_SWIFT_ENDPOINT=<endpoint>
OPENDAL_SWIFT_CONTAINER=<container>
OPENDAL_SWIFT_ROOT=/path/to/dir
OPENDAL_SWIFT_TOKEN=<token>
# gridfs
OPENDAL_GRIDFS_CONNECTION_STRING=mongodb://localhost:27017
OPENDAL_GRIDFS_DATABASE=<database>
OPENDAL_GRIDFS_BUCKET=<fs>
OPENDAL_GRIDFS_CHUNK_SIZE=<chunk_size>
# alluxio
OPENDAL_ALLUXIO_ENDPOINT=<endpoint>
OPENDAL_ALLUXIO_ROOT=/path/to/dor
# b2
OPENDAL_B2_ROOT=/path/to/dir
OPENDAL_B2_BUCKET=<bucket>
OPENDAL_B2_BUCKET_ID=<bucket_id>
OPENDAL_B2_APPLICATION_KEY_ID=<key_id>
OPENDAL_B2_APPLICATION_KEY=<application_key>
# huggingface
OPENDAL_HUGGINGFACE_REPO_TYPE=dataset
OPENDAL_HUGGINGFACE_REPO_ID=opendal/huggingface-testdata
OPENDAL_HUGGINGFACE_REVISION=main
OPENDAL_HUGGINGFACE_ROOT=/testdata/
# seafile
OPENDAL_SEAFILE_ROOT=/path/to/dir
OPENDAL_SEAFILE_ENDPOINT=<endpoint>
OPENDAL_SEAFILE_USERNAME=<usernmae>
OPENDAL_SEAFILE_PASSWORD=<password>
OPENDAL_SEAFILE_REPO_NAME=<repo_name>
# upyun
OPENDAL_UPYUN_ROOT=/path/to/dir
OPENDAL_UPYUN_BUCKET=<bucket>
OPENDAL_UPYUN_OPERATOR=<operator>
OPENDAL_UPYUN_PASSWORD=<password>
# pcloud
OPENDAL_PCLOUD_ROOT=/path/to/dir
OPENDAL_PCLOUD_ENDPOINT=<endpoint>
OPENDAL_PCLOUD_USERNAME=<username>
OPENDAL_PCLOUD_PASSWORD=<password>
# yandex-disk
OPENDAL_YANDEX_DISK_ROOT=/path/to/dir
OPENDAL_YANDEX_DISK_ACCESS_TOKEN=<access_token>
# koofr
OPENDAL_KOOFR_ROOT=/path/to/dir
OPENDAL_KOOFR_ENDPOINT=<endpoint>
OPENDAL_KOOFR_EMAIL=<email>
OPENDAL_KOOFR_PASSWORD=<password>
# vercel blob
OPENDAL_VERCEL_BLOB_ROOT=/path/to/dir
OPENDAL_VERCEL_BLOB_TOKEN=<token>
# aliyun drive
OPENDAL_ALIYUN_DRIVE_ROOT=/path/to/dir
OPENDAL_ALIYUN_DRIVE_REFRESH_TOKEN=<refresh_token>
OPENDAL_ALIYUN_DRIVE_CLIENT_ID=<client_id>
OPENDAL_ALIYUN_DRIVE_CLIENT_SECRET=<client_secret>
# cloudflare
OPENDAL_CLOUDFLARE_KV_ROOT=/path/to/dir
OPENDAL_CLOUDFLARE_KV_API_TOKEN=<api_token>
OPENDAL_CLOUDFLARE_KV_ACCOUNT_ID=<account_id>
OPENDAL_CLOUDFLARE_KV_NAMESPACE_ID=<namespace_id>
```
--------------------------------------------------------------------------------
/bindings/nodejs/npm/darwin-x64/README.md:
--------------------------------------------------------------------------------
```markdown
# `@opendal/lib-darwin-x64`
This is the **x86_64-apple-darwin** binary for `opendal`
```
--------------------------------------------------------------------------------
/bindings/nodejs/npm/darwin-arm64/README.md:
--------------------------------------------------------------------------------
```markdown
# `@opendal/lib-darwin-arm64`
This is the **aarch64-apple-darwin** binary for `opendal`
```
--------------------------------------------------------------------------------
/bindings/nodejs/npm/win32-x64-msvc/README.md:
--------------------------------------------------------------------------------
```markdown
# `@opendal/lib-win32-x64-msvc`
This is the **x86_64-pc-windows-msvc** binary for `opendal`
```
--------------------------------------------------------------------------------
/bindings/nodejs/npm/linux-x64-gnu/README.md:
--------------------------------------------------------------------------------
```markdown
# `@opendal/lib-linux-x64-gnu`
This is the **x86_64-unknown-linux-gnu** binary for `opendal`
```
--------------------------------------------------------------------------------
/bindings/nodejs/npm/linux-x64-musl/README.md:
--------------------------------------------------------------------------------
```markdown
# `@opendal/lib-linux-x64-musl`
This is the **x86_64-unknown-linux-musl** binary for `opendal`
```
--------------------------------------------------------------------------------
/bindings/nodejs/npm/win32-arm64-msvc/README.md:
--------------------------------------------------------------------------------
```markdown
# `@opendal/lib-win32-arm64-msvc`
This is the **aarch64-pc-windows-msvc** binary for `opendal`
```
--------------------------------------------------------------------------------
/bindings/nodejs/npm/linux-arm64-gnu/README.md:
--------------------------------------------------------------------------------
```markdown
# `@opendal/lib-linux-arm64-gnu`
This is the **aarch64-unknown-linux-gnu** binary for `opendal`
```
--------------------------------------------------------------------------------
/bindings/nodejs/npm/linux-arm64-musl/README.md:
--------------------------------------------------------------------------------
```markdown
# `@opendal/lib-linux-arm64-musl`
This is the **aarch64-unknown-linux-musl** binary for `opendal`
```
--------------------------------------------------------------------------------
/core/edge/README.md:
--------------------------------------------------------------------------------
```markdown
# OpenDAL Edge Tests
OpenDAL edge tests served as edge tests for the OpenDAL project. They will have pre-set data and will test the functionality of the OpenDAL project.
```
--------------------------------------------------------------------------------
/.github/scripts/weekly_update/README.md:
--------------------------------------------------------------------------------
```markdown
## OpenDAL Weekly Update
```shell
uv run -s main.py apache/opendal --output weekly_update.md
```
Available options:
- `GITHUB_TOKEN`
- `OPENAI_API_KEY`
- `OPENAI_API_BASE`
- `OPENAI_MODEL`
```
--------------------------------------------------------------------------------
/core/benches/types/README.md:
--------------------------------------------------------------------------------
```markdown
# Types Benchmark Tests
This benchmark contains performance testing of critical data structures in opendal types, currently including performance testing of Buffer.
## Run
```shell
cargo bench types --features tests
```
```
--------------------------------------------------------------------------------
/dev/README.md:
--------------------------------------------------------------------------------
```markdown
# Apache OpenDAL™ Dev
This project is designed to provide dev tools for the entire opendal project.
## Usage
List all available recipes:
```bash
just --list
```
## Recipes
### Generate
Generate code for OpenDAL services.
```bash
just generate python
```
```
--------------------------------------------------------------------------------
/core/edge/file_write_on_full_disk/README.md:
--------------------------------------------------------------------------------
```markdown
# File Write on Fill Disk
Reported by [The `FsBackend` only writes partial bytes and doesn't raise any errors](https://github.com/apache/opendal/issues/3052).
Setup:
```shell
fallocate -l 512K disk.img
mkfs disk.img
mkdir ./td
sudo mount -o loop td.img ./td
chmod a+wr ./td
```
```
--------------------------------------------------------------------------------
/core/benches/README.md:
--------------------------------------------------------------------------------
```markdown
# OpenDAL Benchmarks
Running benchmark
```shell
OPENDAL_TEST=memory cargo bench
```
Build flamegraph:
```shell
cargo flamegraph --bench io -- seek --bench
```
- `--bench io` pick the `io` target.
- `-- seek --bench`: chose the benches with `seek` and `--bench` is required by `criterion`
After `flamegraph.svg` has been generated, use browser to visit it.
```
--------------------------------------------------------------------------------
/bindings/nodejs/benchmark/README.md:
--------------------------------------------------------------------------------
```markdown
# OpenDAL Node.js Bindings Benchmark
This benchmark is test against the opendal and aws js sdk.
To run the benchmark, please make sure the following env have been set correctly.
- AWS_S3_ENDPOINT: the endpoint of the s3 service
- AWS_S3_REGION: the region of the s3 service
- AWS_ACCESS_KEY_ID: the access key of the s3 service
- AWS_SECRET_ACCESS_KEY: the secret key of the s3 service
- AWS_BUCKET: the bucket name of the s3 service
To run the benchmark:
```shell
pnpm bench
```
```
--------------------------------------------------------------------------------
/core/edge/s3_aws_assume_role_with_web_identity/README.md:
--------------------------------------------------------------------------------
```markdown
# S3 AWS Assume Role With Web Identity
This edge test case is for AWS s3 services that authed by `Assume Role With Web Identity`.
For setup, please configure bucket and OIDC correctly, for example:
```yaml
- uses: actions/github-script@v7
id: id-token
with:
script: return await core.getIDToken("sts.amazonaws.com")
result-encoding: string
- name: Write ID token to file
run: echo "${{ steps.id-token.outputs.result }}" > core/tests/data/web_identity_token
```
And configure `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN`.
```
--------------------------------------------------------------------------------
/core/benches/ops/README.md:
--------------------------------------------------------------------------------
```markdown
# Ops Benchmark Tests
Ops Benchmark Tests measure every operation's performance on the target platform.
To support benching different backends simultaneously, we use `environment value` to carry the backend config.
## Setup
Please copy `.env.example` to `.env` and change the values on need.
Take `fs` for example, we add config on `fs` on `/tmp`.
```dotenv
OPENDAL_FS_ROOT=/tmp
```
Notice: The default will skip benches if the env is not set.
## Run
Test specific backend, take s3 for example, first set the corresponding environment variables of s3, then:
```shell
OPENDAL_TEST=s3
cargo bench ops --features tests
```
```
--------------------------------------------------------------------------------
/core/edge/s3_read_on_wasm/README.md:
--------------------------------------------------------------------------------
```markdown
# S3 Read on WASM
This test is used to ensure opendal can run on WASM target.
## Setup
Start the S3 server at `http://127.0.0.1:9900`
We are using the following config values, please feel free to change based on your own needs.
```rust
let mut cfg = S3::default();
cfg.endpoint("http://127.0.0.1:9900");
cfg.access_key_id("minioadmin");
cfg.secret_access_key("minioadmin");
cfg.bucket("opendal");
cfg.region("us-east-1");
```
## Install
```shell
cargo install wasm-pack
```
## Build
```shell
wasm-pack build
```
## Test
NOTE:
- You need to have Chrome installed.
- We can't run on node.js yet.
- We can't run on headless chrome yet.
```shell
wasm-pack test --chrome
```
```
--------------------------------------------------------------------------------
/bindings/python/benchmark/README.md:
--------------------------------------------------------------------------------
```markdown
# OpenDAL Python Bindings Benchmark
This benchmark is test against the opendal and aws python sdk.
To run the benchmark, please make sure the following env have been set correctly.
- AWS_REGION: the region of the s3 service
- AWS_ENDPOINT: the endpoint of the s3 service
- AWS_ACCESS_KEY_ID: the access key of the s3 service
- AWS_SECRET_ACCESS_KEY: the secret key of the s3 service
- AWS_S3_BUCKET: the bucket name of the s3 service
To run the benchmark:
```shell
maturin develop -r -E=benchmark
export AWS_ENDPOINT=http://127.0.0.1:9000
export AWS_REGION=us-east-1
export AWS_ACCESS_KEY_ID=minioadmin
export AWS_SECRET_ACCESS_KEY=minioadmin
export AWS_S3_BUCKET=opendal
uv run async_opendal_benchmark.py
uv run async_origin_s3_benchmark_with_gevent.py
```
```
--------------------------------------------------------------------------------
/fixtures/redis/ssl/README.md:
--------------------------------------------------------------------------------
```markdown
# Key Maintenance
## Check Cert
```shell
openssl x509 -in ca.crt -text -noout
```
## Generate a new CA Cert
```shell
# Generate a new CA key
openssl genrsa -out ca.key 2048
# Generate a new CA cert which valid for 100 years
openssl req -x509 -new -nodes -key ca.key -sha256 -days 36500 -out ca.crt -subj "/C=US/O=Apache OpenDAL Service Test Redis/CN=redis.test.service.opendal.apache.org"
# Check the cert
openssl x509 -in ca.crt -text -noout
```
## Generate a new Peer Cert
```shell
# Generate a new perr key
openssl genrsa -out redis.key 2048
# Generate a new CSR
openssl req -new -key redis.key -out redis.csr -config req.conf
# Use CA Cert to sign the CSR
openssl x509 -req -in redis.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out redis.crt -days 36500 -sha256 -extfile req.conf -extensions v3_ca
# Check the cert
openssl x509 -in redis.crt -text -noout
```
```
--------------------------------------------------------------------------------
/core/benches/vs_fs/README.md:
--------------------------------------------------------------------------------
```markdown
# OpenDAL Benchmark VS Fs
This benchmark compares the performance of OpenDAL with the performance of the `std::fs`.
## Goal
We expect OpenDAL to match `std::fs` in speed: the throughput of OpenDAL should be within a `5%` range of `std::fs`.
## Usage
For test: `cargo run`
```shell
Testing vs_fs/std_fs_read
Success
Testing vs_fs/opendal_fs_read
Success
Testing vs_fs/opendal_fs_read_with_range
Success
```
For bench: `cargo run --release -- --bench`
```shell
read/std_fs time: [749.57 µs 762.69 µs 777.07 µs]
thrpt: [20.108 GiB/s 20.487 GiB/s 20.845 GiB/s]
read/opendal_fs time: [750.90 µs 755.39 µs 760.49 µs]
thrpt: [20.546 GiB/s 20.685 GiB/s 20.808 GiB/s]
read/opendal_fs_with_range
time: [684.02 µs 690.77 µs 697.99 µs]
thrpt: [22.386 GiB/s 22.620 GiB/s 22.843 GiB/s]
```
```
--------------------------------------------------------------------------------
/scripts/README.md:
--------------------------------------------------------------------------------
```markdown
# OpenDAL Scripts
This module provides scripts to make maintainers lives easier.
OpenDAL users don't need to care about this folder.
NOTES: all scripts must be running at root folder of OpenDAL project.
## Release
```shell
just release
```
> Before running release, please make sure you have bump all versions.
### Preparations
Import gpg key
```shell
curl https://downloads.apache.org/opendal/KEYS > KEYS # Download KEYS
gpg --import KEYS # Import KEYS to local
```
Trust the public key
```shell
$ gpg --edit-key xxxxxxxxxx #KEY user used in this version
gpg (GnuPG) 2.2.21; Copyright (C) 2020 Free Software Foundation, Inc.
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Secret key is available.
gpg> trust #trust
Please decide how far you trust this user to correctly verify other users' keys
(by looking at passports, checking fingerprints from different sources, etc.)
1 = I don't know or won't say
2 = I do NOT trust
3 = I trust marginally
4 = I trust fully
5 = I trust ultimately
m = back to the main menu
Your decision? 5 #choose 5
Do you really want to set this key to ultimate trust? (y/N) y #choose y
```
## Verify
```shell
./scripts/verify.py
```
```
--------------------------------------------------------------------------------
/core/fuzz/README.md:
--------------------------------------------------------------------------------
```markdown
# Fuzz Test for OpenDAL
fuzz test are used to test the robustness of the code.
## Setup
To run the fuzz tests, please copy the `.env.example`, which is at project root, to `.env` and change the values on need.
Take `fs` for example, we need to change to enable fuzz test on `fs` on `/tmp`.
```dotenv
OPENDAL_TEST=fs
OPENDAL_FS_ROOT=/path/to/dir/
```
into
```dotenv
OPENDAL_TEST=fs
OPENDAL_FS_ROOT=/tmp/
```
## Run
List all fuzz targets.
```bash
cargo +nightly fuzz list
```
Build fuzz targets.
```bash
cargo +nightly fuzz build
```
Run a fuzz target(such as `reader`).
```bash
cargo +nightly fuzz run fuzz_reader
```
## Crash Reproduction
If you want to reproduce a crash, you first need to obtain the Base64 encoded code, which usually appears at the end of a crash report, and store it in a file.
Alternatively, if you already have the crash file, you can skip this step.
```bash
echo "Base64" > .crash
```
Print the `std::fmt::Debug` output for an input.
```bash
cargo +nightly fuzz fmt fuzz_target .crash
```
Rerun the fuzz test with the input.
```bash
cargo +nightly fuzz run fuzz_target .crash
```
For more details, please visit [cargo fuzz](https://rust-fuzz.github.io/book/cargo-fuzz/tutorial.html) or run the command cargo fuzz --help.
```
--------------------------------------------------------------------------------
/core/benches/vs_s3/README.md:
--------------------------------------------------------------------------------
```markdown
# OpenDAL Benchmark VS AWS SDK S3
This benchmark compares the performance of OpenDAL with the performance of the `aws_sdk_s3`.
## Goal
We expect OpenDAL to match `aws_sdk_s3` in speed: the throughput of OpenDAL should be within a `5%` range of `aws_sdk_s3`.
## Notes
Please run bench case separately, otherwise the result will be affected by each other since we are sharing the same runtime.
## Usage
For test: `cargo run`
```shell
> cargo run
Testing read/opendal_s3_reader
Success
Testing read/aws_s3_sdk_into_async_read
Success
Testing read/opendal_s3_reader_with_capacity
Success
Testing read/aws_s3_sdk_into_async_read_with_capacity
Success
```
For bench case: `cargo run --release -- opendal_s3_reader --bench`
```shell
> cargo run --release -- opendal_s3_reader --bench
read/opendal_s3_reader time: [12.773 ms 13.004 ms 13.232 ms]
thrpt: [1.1809 GiB/s 1.2016 GiB/s 1.2232 GiB/s]
```
For bench: `cargo run --release -- --bench`
```shell
> cargo run --release -- --bench
read/opendal_s3_reader time: [12.773 ms 13.004 ms 13.232 ms]
thrpt: [1.1809 GiB/s 1.2016 GiB/s 1.2232 GiB/s]
read/aws_s3_sdk_into_async_read
time: [12.527 ms 12.842 ms 13.158 ms]
thrpt: [1.1875 GiB/s 1.2168 GiB/s 1.2473 GiB/s]
read/opendal_s3_reader_with_capacity
time: [9.6098 ms 9.8133 ms 10.017 ms]
thrpt: [1.5599 GiB/s 1.5922 GiB/s 1.6259 GiB/s]
read/aws_s3_sdk_into_async_read_with_capacity
time: [9.8970 ms 10.113 ms 10.329 ms]
thrpt: [1.5128 GiB/s 1.5451 GiB/s 1.5788 GiB/s]
```
```
--------------------------------------------------------------------------------
/bindings/README.md:
--------------------------------------------------------------------------------
```markdown
# OpenDAL Bindings
This folder contains the bindings for OpenDAL. Currently, we support the following bindings:
### Released Bindings
* [Java](java/README.md)
* [Node.js](nodejs/README.md)
* [Python](python/README.md)
### Unreleased Bindings
* [C](c/README.md)
* [C++](cpp/README.md)
* [C#](dotnet/README.md)
* [D](d/README.md)
* [Dart](dart/README.md)
* [.NET](dotnet/README.md)
* [Go](go/README.md)
* [Haskell](haskell/README.md)
* [Lua](lua/README.md)
* [OCaml](ocaml/README.md)
* [PHP](php/README.md)
* [Ruby](ruby/README.md)
* [Swift](swift/README.md)
* [Zig](zig/README.md)
## Versioning
**Important**: Each binding has its own independent version number, which may differ from the Rust core version. This allows bindings to be released and versioned independently based on their own release cycles and compatibility requirements.
For example, while the Rust core might be at version `0.55.0`, a binding might be at version `0.47.0` or `0.49.2`. When checking for updates or compatibility, always refer to the specific binding's version rather than the core version.
## Getting Started
Every binding should provide a `README.md` file to help users get started.
The `README.md` file should contain the following sections:
* **Installation**: how to install the binding.
* **Usage**: how to use the binding.
* **Development**: how to develop the binding.
* **Testing**: how to test the binding.
You can find the `README.md` file for each binding in the corresponding folder.
Please refer to the bindings listed above or which are already released for more details.
## Contributing
We welcome contributions to OpenDAL. Please refer to [CONTRIBUTING.md](../CONTRIBUTING.md) for the contributing guidelines.
```
--------------------------------------------------------------------------------
/core/tests/behavior/README.md:
--------------------------------------------------------------------------------
```markdown
# Behavior Test for OpenDAL
Behavior tests are used to make sure every service works correctly.
To support different testing backends simultaneously, we use `environment value` to carry the backend config.
## Setup
### Install Docker
Currently, opendal uses Docker to set up environment for behaviour tests. If you are a MacOS user, The docker does not work properly by default(Docker Desktop) in the MacOS environment. It is strongly recommended to use **OrbStack** instead of Docker Desktop. You can install orbstack using Homebrew.
```shell
brew install orbstack
```
### Prepare Env
To run the behavior tests, please copy the `.env.example`, which is at project root, to `.env` and change the values on need.
Take `fs` for example, we need to change to enable behavior test on `fs` on `/tmp`.
```dotenv
OPENDAL_FS_ROOT=/path/to/dir/
```
into
```doten
OPENDAL_FS_ROOT=/tmp/
```
Notice: If the env variables are not set, all behavior tests will be skipped by default.
## Run
Use `OPENDAL_TEST` to control which service to test:
```shell
OPENDAL_TEST=fs cargo test behavior --features tests
```
To run certain types of tests(such as `write`), we use `behavior::test_write`.
```shell
OPENDAL_TEST=fs cargo test behavior::test_write --features tests
```
You can also run specific test(such as `test_stat_dir`) for specific backend.
```shell
OPENDAL_TEST=fs cargo test behavior::test_stat_dir --features tests
```
## Debug
To debug a behavior test, you can:
- Add env `RUST_LOG=debug` to enable logging
- Add env `RUST_BACKTRACE=full` to enable the full backtrace
- Add `--show-output` to show the whole output even when test succeeded.
Take `memory` service as an example, the full command will be:
```shell
RUST_LOG=debug RUST_BACKTRACE=full OPENDAL_TEST=memory cargo test behavior --features tests -- --show-output
```
You use `export` to avoid set env every time:
```shell
export RUST_LOG=debug
export RUST_BACKTRACE=full
OPENDAL_TEST=memory cargo test behavior --features tests -- --show-output
```
For more details, please visit [cargo test](https://doc.rust-lang.org/cargo/commands/cargo-test.html) or run the command `cargo test --help`.
```
--------------------------------------------------------------------------------
/bindings/python/README.md:
--------------------------------------------------------------------------------
```markdown
# Apache OpenDAL™ Python Binding
[](https://pypi.org/project/opendal/)
[](https://pypi.org/project/opendal/)
[](https://opendal.apache.org/docs/python/)
This package provides a native Python binding for **Apache OpenDAL™**, a data access
layer that allows you to access various storage services in a unified way.
> **Note**: This binding has its own independent version number, which may differ from the Rust core version. When checking for updates or compatibility, always refer to this binding's version rather than the core version.
## Useful Links
- [Documentation](https://opendal.apache.org/docs/python/)
- [Examples](./docs/examples)
- [Upgrade Guide](./upgrade.md)
---
## Features
- **Unified API**: Access S3, GCS, Azure Blob, HDFS, FTP, and more with the same set of
commands.
- **Native Performance**: Built in Rust for high performance and safety.
- **Async Support**: First-class `async` API for modern Python applications.
- **Easy to Use**: Simple and intuitive API design.
---
## Installation
Install the package directly from PyPI:
```bash
pip install opendal
```
---
## Usage
Here are a few examples of how to use OpenDAL with different storage backends.
### Local Filesystem (`fs`)
```python
import opendal
# Initialize the operator for the local filesystem
op = opendal.Operator("fs", root="/tmp")
# Write data to a file
op.write("test.txt", b"Hello World")
# Read data from the file
content = op.read("test.txt")
print(op.read("test.txt"))
# Get metadata
metadata = op.stat("test.txt")
print(f"Content length: {metadata.content_length}") # Output: 11
```
### Amazon S3
The API remains the same—just change the scheme and credentials.
```python
import opendal
# Initialize the operator for S3
op = opendal.Operator(
"s3",
bucket="your_bucket_name",
region="your_region",
root="/path/to/root"
)
op.write("test.txt", b"Hello World")
print(op.read("test.txt"))
print(op.stat("test.txt").content_length)
```
### Async Usage (`s3`)
OpenDAL also provides a fully asynchronous API.
```python
import asyncio
import opendal
async def main():
# Use AsyncOperator for async operations
op = opendal.AsyncOperator("s3", root="/tmp", bucket="your_bucket_name", region="your_region")
await op.write("test.txt", b"Hello World")
print(await op.read("test.txt"))
asyncio.run(main())
```
---
## Development
This project uses [`just`](https://github.com/casey/just) as a command runner to
simplify the development workflow.
1. **Clone the repository and set up the environment:**
```shell
# This will create a virtual environment and install all dependencies
just setup
```
2. **Run tests:**
```shell
# Example: Run tests for the 'fs' operator
OPENDAL_TEST=fs OPENDAL_FS_ROOT=/tmp just test
```
For a complete guide on building, testing, and contributing, please see our
**[CONTRIBUTING.md](./CONTRIBUTING.md)** file.
---
## Used By
Check out the [users list](./users.md) for more details on who is using OpenDAL.
## License and Trademarks
Licensed under the Apache License, Version 2.0:
http://www.apache.org/licenses/LICENSE-2.0
Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of
the Apache Software Foundation.
```
--------------------------------------------------------------------------------
/core/core/src/docs/rfcs/README.md:
--------------------------------------------------------------------------------
```markdown
# RFCs - OpenDAL Active RFC List
RFCs power OpenDAL's development.
The "RFC" (request for comments) process is intended to provide a consistent and controlled path for changes to OpenDAL (such as new features) so that all stakeholders can be confident about the direction of the project.
Many changes, including bug fixes and documentation improvements, can be implemented and reviewed via the normal GitHub pull request workflow.
Some changes, though, are "substantial" and we ask that these be put through a bit of a design process and produce a consensus among the OpenDAL community.
### Which kinds of changes require an RFC?
Any substantial change or addition to the project that would require a significant amount of work to implement should generally be an RFC.
Some examples include:
- A new feature that creates a new public API or raw API.
- The removal of features that already shipped as part of the release.
- A big refactor of existing code or reorganization of code into new modules.
Those are just a few examples. Ultimately, the judgment call of what constitutes a big enough change to warrant an RFC is left to the project maintainers.
If you submit a pull request to implement a new feature without going through the RFC process, it may be closed with a polite request to submit an RFC first.
## Before creating the RFC
Preparing in advance before submitting an RFC hastily can increase its chances of being accepted. If you have proposals to make, it is advisable to engage in some preliminary groundwork to facilitate a smoother process.
It is great to seek feedback from other project developers first, as this can help validate the viability of the RFC. To ensure a sustained impact on the project, it is important to work together and reach a consensus.
Common preparatory steps include presenting your idea on platforms such as GitHub [issues](https://github.com/apache/opendal/issues/) or [discussions](https://github.com/apache/opendal/discussions/categories/ideas), or engaging in discussions through our [email list](https://opendal.apache.org/community/#mailing-list) or [Discord server](https://opendal.apache.org/discord).
## The RFC process
- Fork the [OpenDAL repo](https://github.com/apache/opendal) and create your branch from `main`.
- Copy [`0000_example.md`] to `0000-my-feature.md` (where "my-feature" is descriptive). Don't assign an RFC number yet; This is going to be the PR number, and we'll rename the file accordingly if the RFC is accepted.
- Submit a pull request. As a pull request, the RFC will receive design feedback from the larger community, and the author should be prepared to revise it in response.
- Now that your RFC has an open pull request, use the issue number of this PR to update your `0000-` prefix to that number.
- Build consensus and integrate feedback. RFCs that have broad support are much more likely to make progress than those that don't receive any comments. Feel free to reach OpenDAL maintainers for help.
- RFCs rarely go through this process unchanged, especially as alternatives and drawbacks are shown. You can make edits, big and small, to the RFC to clarify or change the design, but make changes as new commits to the pull request, and leave a comment on the pull request explaining your changes. Specifically, do not squash or rebase commits after they are visible on the pull request.
- The RFC pull request lasts for three days after the last update. After that, the RFC will be accepted or declined based on the consensus reached in the discussion.
- For the accepting of an RFC, we will require approval from at least three maintainers.
- Once the RFC is accepted, please create a tracking issue and update links in RFC. And then the PR will be merged and the RFC will become 'active' status.
## Implementing an RFC
An active RFC does not indicate the priority assigned to its implementation,
nor does it imply that a developer has been specifically assigned the task of implementing the feature.
The RFC author is encouraged to submit an implementation after the RFC has been accepted.
Nevertheless, it is not obligatory for them to do so.
Accepted RFCs may represent features that can wait until a developer chooses to work on them.
Each accepted RFC is associated with an issue in the OpenDAL repository, which tracks its implementation.
If you are interested in implementing an RFC but are unsure if someone else is already working on it,
feel free to inquire by leaving a comment on the associated issue.
## Some useful tips
- The author of an RFC may not be the same one as the implementer. Therefore, when submitting an RFC, it is advisable to include sufficient information.
- If modifications are needed for an accepted RFC, please submit a new pull request or create a new RFC to propose changes.
```
--------------------------------------------------------------------------------
/bindings/java/README.md:
--------------------------------------------------------------------------------
```markdown
# Apache OpenDAL™ Java Bindings
[](https://central.sonatype.com/search?q=opendal&smo=true)
[](https://central.sonatype.com/search?q=opendal&smo=true)
[](https://opendal.apache.org/docs/java/)
> **Note**: This binding has its own independent version number, which may differ from the Rust core version. When checking for updates or compatibility, always refer to this binding's version rather than the core version.
## Useful Links
- [Documentation](https://opendal.apache.org/docs/java/)
- [Upgrade Guide](./upgrade.md)
## Example
```java
import java.util.HashMap;
import java.util.Map;
import org.apache.opendal.AsyncOperator;
import org.apache.opendal.Operator;
public class Main {
public static void main(String[] args) {
final Map<String, String> conf = new HashMap<>();
conf.put("root", "/tmp");
try (AsyncOperator op = AsyncOperator.of("fs", conf)) {
op.write("/path/to/data", "Hello world").join();
System.out.println(new String(op.read("/path/to/data").join()));
}
}
}
```
## Getting Started
This project is built upon the native OpenDAL lib. And it is released for multiple platforms that you can use a classifier to specify the platform you are building the application on.
### Maven
Generally, you can first add the `os-maven-plugin` for automatically detect the classifier based on your platform:
```xml
<build>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>1.7.0</version>
</extension>
</extensions>
</build>
```
Then add the dependency to `opendal` as following:
```xml
<dependencies>
<dependency>
<groupId>org.apache.opendal</groupId>
<artifactId>opendal</artifactId>
<version>${opendal.version}</version>
</dependency>
<dependency>
<groupId>org.apache.opendal</groupId>
<artifactId>opendal</artifactId>
<version>${opendal.version}</version>
<classifier>${os.detected.classifier}</classifier>
</dependency>
</dependencies>
```
### Gradle
For Gradle, you can first add the `com.google.osdetector` for automatically detect the classifier based on your platform:
```groovy
plugins {
id "com.google.osdetector" version "1.7.3"
}
```
Then add the dependency to `opendal as following:
```groovy
dependencies {
implementation "org.apache.opendal:opendal:$opendalVersion"
implementation "org.apache.opendal:opendal:$opendalVersion:$osdetector.classifier"
}
```
### Classified library
Note that the dependency without classifier ships all classes and resources except the "opendal_java" shared library. And those with classifier bundle only the shared library.
For downstream usage, it's recommended:
* Depend on the one without classifier to write code;
* Depend on the classified ones with "test" for testing.
To load the shared library correctly, you can choose one of the following approaches:
* Append the classified JARs to the classpath at the runtime;
* Depend on the classified JARs and build a fat JAR (You may need to depend on all the provided classified JARs for running on multiple platforms);
* Build your own "opendal_java" shared library and specify "-Djava.library.path" to the folder containing that shared library.
## Build
This project provides OpenDAL Java bindings with artifact name `opendal`. It depends on JDK 8 or later.
You can use Maven to build both Rust dynamic lib and JAR files with one command now:
```shell
./mvnw clean package -DskipTests=true
```
## Run tests
Currently, all tests are written in Java.
You can run the base tests with the following command:
```shell
./mvnw clean verify
```
## Code style
This project uses [spotless](https://github.com/diffplug/spotless) for code formatting so that all developers share a consistent code style without bikeshedding on it.
You can apply the code style with the following command::
```shell
./mvnw spotless:apply
```
## Run behavior tests
Services behavior tests read necessary configs from env vars or the `.env` file.
You can copy [.env.example](/.env.example) to `${project.rootdir}/.env` and change the values on need, or directly set env vars with `export KEY=VALUE`.
Take `fs` for example, we need to enable bench on `fs` on `/tmp/`:
```properties
OPENDAL_TEST=fs
OPENDAL_FS_ROOT=/tmp/
```
You can run service behavior tests of enabled with the following command:
```shell
./mvnw test -Dtest="behavior.*Test"
```
Remember to enable the necessary features via `-Dcargo-build.features=services-xxx` when running specific service test:
```shell
export OPENDAL_TEST=redis
export OPENDAL_REDIS_ENDPOINT=tcp://127.0.0.1:6379
export OPENDAL_REDIS_ROOT=/
export OPENDAL_REDIS_DB=0
./mvnw test -Dtest="behavior.*Test" -Dcargo-build.features=services-redis
```
## Used by
Check out the [users](./users.md) list for more details on who is using OpenDAL.
## License and Trademarks
Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation.
```
--------------------------------------------------------------------------------
/bindings/nodejs/README.md:
--------------------------------------------------------------------------------
```markdown
# Apache OpenDAL™ Node.js Binding
[](https://www.npmjs.com/package/opendal)
[](https://www.npmjs.com/package/opendal)
[](https://opendal.apache.org/docs/nodejs/)
> **Note**: This binding has its own independent version number, which may differ from the Rust core version. When checking for updates or compatibility, always refer to this binding's version rather than the core version.
## Useful Links
- [Documentation](https://opendal.apache.org/docs/nodejs/)
- [Upgrade Guide](./upgrade.md)
## Installation
```shell
npm install opendal
```
## Docs
To build the docs locally, please run the following commands:
```shell
# Only need to run once unless you want to update the docs theme
pnpm run build:theme
# Build the docs
pnpm run docs
```
## Tests
Services behavior tests read necessary configs from env vars or the `.env` file.
You can copy [.env.example](/.env.example) to `$(pwd)/.env` and change the values on need, or directly set env vars with `export KEY=VALUE`.
Take `fs` for example, we need to enable bench on `fs` on `/tmp`:
```properties
OPENDAL_TEST=fs
OPENDAL_FS_ROOT=/tmp
```
You can run service behavior tests of enabled with the following command:
```shell
pnpm build && pnpm test
```
## Usage
```javascript
import { Operator } from "opendal";
async function main() {
const op = new Operator("fs", { root: "/tmp" });
await op.write("test", "Hello, World!");
const bs = await op.read("test");
console.log(new TextDecoder().decode(bs));
const meta = await op.stat("test");
console.log(`contentLength: ${meta.contentLength}`);
}
main();
```
## Layers
OpenDAL provides layers to add additional functionality to operators. You can chain multiple layers together to build powerful data access pipelines.
### Available Layers
- **RetryLayer** - Retry failed operations automatically
- **ConcurrentLimitLayer** - Limit concurrent requests
- **TimeoutLayer** - Add timeout for operations
- **LoggingLayer** - Log all operations
- **ThrottleLayer** - Limit bandwidth usage
### TimeoutLayer
Prevents operations from hanging indefinitely:
```javascript
import { Operator, TimeoutLayer } from "opendal";
const op = new Operator("fs", { root: "/tmp" });
const timeout = new TimeoutLayer();
timeout.timeout = 10000; // 10 seconds for non-IO operations (ms)
timeout.ioTimeout = 3000; // 3 seconds for IO operations (ms)
op.layer(timeout.build());
```
**Default values:** timeout: 60s, ioTimeout: 10s
### LoggingLayer
Add structured logging for debugging and monitoring:
```javascript
import { Operator, LoggingLayer } from "opendal";
const op = new Operator("fs", { root: "/tmp" });
const logging = new LoggingLayer();
op.layer(logging.build());
// All operations will be logged
await op.write("test.txt", "Hello World");
```
Enable logging output:
```bash
# Show debug logs
RUST_LOG=debug node app.js
# Show only OpenDAL logs
RUST_LOG=opendal::services=debug node app.js
```
### ThrottleLayer
Control bandwidth usage with rate limiting:
```javascript
import { Operator, ThrottleLayer } from "opendal";
const op = new Operator("s3", {
bucket: "my-bucket",
region: "us-east-1",
});
// Limit to 10 KiB/s with 10 MiB burst
const throttle = new ThrottleLayer(10 * 1024, 10 * 1024 * 1024);
op.layer(throttle.build());
```
**Parameters:**
- `bandwidth`: Maximum bytes per second
- `burst`: Maximum burst size (must be larger than any operation size)
### RetryLayer
Automatically retry temporary failed operations:
```javascript
import { Operator, RetryLayer } from "opendal";
const op = new Operator("s3", {
bucket: "my-bucket",
region: "us-east-1",
});
const retry = new RetryLayer();
retry.maxTimes = 3;
retry.jitter = true;
op.layer(retry.build());
```
### ConcurrentLimitLayer
Limit concurrent requests to storage services:
```javascript
import { Operator, ConcurrentLimitLayer } from "opendal";
const op = new Operator("s3", {
bucket: "my-bucket",
region: "us-east-1",
});
// Allow max 1024 concurrent operations
const limit = new ConcurrentLimitLayer(1024);
limit.httpPermits = 512; // Limit HTTP requests separately
op.layer(limit.build());
```
### Combining Multiple Layers
Stack multiple layers for comprehensive control:
```javascript
import {
Operator,
LoggingLayer,
TimeoutLayer,
RetryLayer,
ThrottleLayer,
} from "opendal";
const op = new Operator("s3", {
bucket: "my-bucket",
region: "us-east-1",
});
// Layer 1: Logging for observability
const logging = new LoggingLayer();
op.layer(logging.build());
// Layer 2: Timeout protection
const timeout = new TimeoutLayer();
timeout.timeout = 30000;
timeout.ioTimeout = 10000;
op.layer(timeout.build());
// Layer 3: Retry on failures
const retry = new RetryLayer();
retry.maxTimes = 3;
retry.jitter = true;
op.layer(retry.build());
// Layer 4: Bandwidth throttling
const throttle = new ThrottleLayer(100 * 1024, 10 * 1024 * 1024);
op.layer(throttle.build());
// Now the operator has full production-ready protection
await op.write("data.json", JSON.stringify(data));
```
## Usage with Next.js
Config automatically be bundled by [Next.js](https://nextjs.org/docs/app/api-reference/config/next-config-js/serverExternalPackages).
```javascript
/** @type {import('next').NextConfig} */
const nextConfig = {
serverExternalPackages: ["opendal"],
};
module.exports = nextConfig;
```
## Contributing
- Start with [Contributing Guide](CONTRIBUTING.md).
- Submit [Issues](https://github.com/apache/opendal/issues/new) for bug report or feature requests.
- Asking questions in the [Discussions](https://github.com/apache/opendal/discussions/new?category=q-a).
- Talk to community at [Discord](https://opendal.apache.org/discord).
## License and Trademarks
Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation.
```
--------------------------------------------------------------------------------
/core/README.md:
--------------------------------------------------------------------------------
```markdown
# Apache OpenDAL™ Rust Core: One Layer, All Storage.
[![Build Status]][actions] [![Latest Version]][crates.io] [![Crate Downloads]][crates.io] [![chat]][discord]
[build status]: https://img.shields.io/github/actions/workflow/status/apache/opendal/ci_core.yml?branch=main
[actions]: https://github.com/apache/opendal/actions?query=branch%3Amain
[latest version]: https://img.shields.io/crates/v/opendal.svg
[crates.io]: https://crates.io/crates/opendal
[crate downloads]: https://img.shields.io/crates/d/opendal.svg
[chat]: https://img.shields.io/discord/1081052318650339399
[discord]: https://opendal.apache.org/discord
Apache OpenDAL™ is an Open Data Access Layer that enables seamless interaction with diverse storage services.
<img src="https://opendal.apache.org/img/architectural.png" alt="OpenDAL Architectural" width="61.8%" />
## Useful Links
- Documentation: [release](https://docs.rs/opendal/) | [dev](https://opendal.apache.org/docs/rust/opendal/)
- [Examples](./examples)
- [Release Notes](https://docs.rs/opendal/latest/opendal/docs/changelog/index.html)
- [Upgrade Guide](https://docs.rs/opendal/latest/opendal/docs/upgrade/index.html)
- [RFC List](https://docs.rs/opendal/latest/opendal/docs/rfcs/index.html)
## Services
OpenDAL supports the following storage [services](https://docs.rs/opendal/latest/opendal/services/index.html):
| Type | Services |
|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------|
| Standard Storage Protocols | ftp http [sftp] [webdav] |
| Object Storage Services | [azblob] [cos] [gcs] [obs] [oss] [s3] <br> [b2] [openstack_swift] [upyun] [vercel-blob] |
| File Storage Services | fs [alluxio] [azdls] [azfile] [compfs] <br> [dbfs] [gridfs] [hdfs] [hdfs-native] [ipfs] [webhdfs] |
| Consumer Cloud Storage Service | [aliyun-drive] [gdrive] [onedrive] [dropbox] [koofr] <br> [pcloud] [seafile] [yandex-disk] |
| Key-Value Storage Services | [cacache] [cloudflare-kv] [dashmap] memory [etcd] <br> [foundationdb] [persy] [redis] [rocksdb] [sled] <br> [redb] [tikv] |
| Database Storage Services | [d1] [mongodb] [mysql] [postgresql] [sqlite] [surrealdb] |
| Cache Storage Services | [ghac] [memcached] [mini-moka] [moka] [vercel-artifacts] |
| Git Based Storage Services | [huggingface] |
[sftp]: https://datatracker.ietf.org/doc/html/draft-ietf-secsh-filexfer-02
[webdav]: https://datatracker.ietf.org/doc/html/rfc4918
[azblob]: https://azure.microsoft.com/en-us/services/storage/blobs/
[cos]: https://www.tencentcloud.com/products/cos
[gcs]: https://cloud.google.com/storage
[obs]: https://www.huaweicloud.com/intl/en-us/product/obs.html
[oss]: https://www.aliyun.com/product/oss
[s3]: https://aws.amazon.com/s3/
[b2]: https://www.backblaze.com/
[openstack_swift]: https://docs.openstack.org/swift/latest/
[upyun]: https://www.upyun.com/
[vercel-blob]: https://vercel.com/docs/storage/vercel-blob
[alluxio]: https://docs.alluxio.io/os/user/stable/en/api/REST-API.html
[azdls]: https://azure.microsoft.com/en-us/products/storage/data-lake-storage/
[azfile]: https://learn.microsoft.com/en-us/rest/api/storageservices/file-service-rest-api
[compfs]: https://github.com/compio-rs/compio/
[dbfs]: https://docs.databricks.com/en/dbfs/index.html
[gridfs]: https://www.mongodb.com/docs/manual/core/gridfs/
[hdfs]: https://hadoop.apache.org/docs/r3.3.4/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html
[hdfs-native]: https://github.com/Kimahriman/hdfs-native
[ipfs]: https://ipfs.tech/
[webhdfs]: https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html
[aliyun-drive]: https://www.aliyundrive.com/
[gdrive]: https://www.google.com/drive/
[onedrive]: https://www.microsoft.com/en-us/microsoft-365/onedrive/online-cloud-storage
[dropbox]: https://www.dropbox.com/
[koofr]: https://koofr.eu/
[pcloud]: https://www.pcloud.com/
[seafile]: https://www.seafile.com/
[yandex-disk]: https://360.yandex.com/disk/
[cacache]: https://crates.io/crates/cacache
[cloudflare-kv]: https://developers.cloudflare.com/kv/
[dashmap]: https://github.com/xacrimon/dashmap
[etcd]: https://etcd.io/
[foundationdb]: https://www.foundationdb.org/
[persy]: https://crates.io/crates/persy
[redis]: https://redis.io/
[rocksdb]: http://rocksdb.org/
[sled]: https://crates.io/crates/sled
[redb]: https://crates.io/crates/redb
[tikv]: https://tikv.org/
[d1]: https://developers.cloudflare.com/d1/
[mongodb]: https://www.mongodb.com/
[mysql]: https://www.mysql.com/
[postgresql]: https://www.postgresql.org/
[sqlite]: https://www.sqlite.org/
[surrealdb]: https://surrealdb.com/
[ghac]: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows
[memcached]: https://memcached.org/
[mini-moka]: https://github.com/moka-rs/mini-moka
[moka]: https://github.com/moka-rs/moka
[vercel-artifacts]: https://vercel.com/docs/concepts/monorepos/remote-caching
[huggingface]: https://huggingface.co/
## Layers
OpenDAL supports the following storage [layers](https://docs.rs/opendal/latest/opendal/layers/index.html) to extend the behavior:
| Name | Depends | Description |
|---------------------------|--------------------------|---------------------------------------------------------------------------------------|
| [`AsyncBacktraceLayer`] | [async-backtrace] | Add Efficient, logical 'stack' traces of async functions for the underlying services. |
| [`AwaitTreeLayer`] | [await-tree] | Add a Instrument await-tree for actor-based applications to the underlying services. |
| [`BlockingLayer`] | [tokio] | Add blocking API support for non-blocking services. |
| [`ChaosLayer`] | [rand] | Inject chaos into underlying services for robustness test. |
| [`ConcurrentLimitLayer`] | [tokio] | Add concurrent request limit. |
| [`DtraceLayer`] | [probe] | Support User Statically-Defined Tracing(aka USDT) on Linux |
| [`LoggingLayer`] | [log] | Add log for every operations. |
| [`MetricsLayer`] | [metrics] | Add metrics for every operations. |
| [`MimeGuessLayer`] | [mime_guess] | Add `Content-Type` automatically based on the file extension in the operation path. |
| [`FastraceLayer`] | [fastrace] | Add fastrace for every operations. |
| [`OtelMetricsLayer`] | [opentelemetry::metrics] | Add opentelemetry::metrics for every operations. |
| [`OtelTraceLayer`] | [opentelemetry::trace] | Add opentelemetry::trace for every operations. |
| [`PrometheusClientLayer`] | [prometheus_client] | Add prometheus metrics for every operations. |
| [`PrometheusLayer`] | [prometheus] | Add prometheus metrics for every operations. |
| [`RetryLayer`] | [backon] | Add retry for temporary failed operations. |
| [`ThrottleLayer`] | [governor] | Add a bandwidth rate limiter to the underlying services. |
| [`TimeoutLayer`] | [tokio] | Add timeout for every operations to avoid slow or unexpected hang operations. |
| [`TracingLayer`] | [tracing] | Add tracing for every operations. |
[`AsyncBacktraceLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.AsyncBacktraceLayer.html
[async-backtrace]: https://github.com/tokio-rs/async-backtrace
[`AwaitTreeLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.AwaitTreeLayer.html
[await-tree]: https://github.com/risingwavelabs/await-tree
[`BlockingLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.BlockingLayer.html
[tokio]: https://github.com/tokio-rs/tokio
[`ChaosLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.ChaosLayer.html
[rand]: https://github.com/rust-random/rand
[`ConcurrentLimitLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.ConcurrentLimitLayer.html
[`DtraceLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.DtraceLayer.html
[probe]: https://github.com/cuviper/probe-rs
[`LoggingLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.LoggingLayer.html
[log]: https://github.com/rust-lang/log
[`MetricsLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.MetricsLayer.html
[metrics]: https://github.com/metrics-rs/metrics
[`MimeGuessLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.MimeGuessLayer.html
[mime_guess]: https://github.com/abonander/mime_guess
[`FastraceLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.FastraceLayer.html
[fastrace]: https://github.com/fastracelabs/fastrace
[`OtelMetricsLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.OtelMetricsLayer.html
[`OtelTraceLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.OtelTraceLayer.html
[opentelemetry::trace]: https://docs.rs/opentelemetry/latest/opentelemetry/trace/index.html
[`PrometheusClientLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.PrometheusClientLayer.html
[prometheus_client]: https://github.com/prometheus/client_rust
[`PrometheusLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.PrometheusLayer.html
[prometheus]: https://github.com/tikv/rust-prometheus
[`RetryLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.RetryLayer.html
[backon]: https://github.com/Xuanwo/backon
[`ThrottleLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.ThrottleLayer.html
[governor]: https://github.com/boinkor-net/governor
[`TimeoutLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.TimeoutLayer.html
[`TracingLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.TracingLayer.html
[tracing]: https://github.com/tokio-rs/tracing
## Quickstart
```rust
use opendal::Result;
use opendal::layers::LoggingLayer;
use opendal::services;
use opendal::Operator;
#[tokio::main]
async fn main() -> Result<()> {
// Pick a builder and configure it.
let mut builder = services::S3::default();
builder.bucket("test");
// Init an operator
let op = Operator::new(builder)?
// Init with logging layer enabled.
.layer(LoggingLayer::default())
.finish();
// Write data
op.write("hello.txt", "Hello, World!").await?;
// Read data
let bs = op.read("hello.txt").await?;
// Fetch metadata
let meta = op.stat("hello.txt").await?;
let mode = meta.mode();
let length = meta.content_length();
// Delete
op.delete("hello.txt").await?;
Ok(())
}
```
## Examples
| Name | Description |
|---------------------|---------------------------------------------------------------|
| [Basic] | Show how to use opendal to operate storage service. |
| [Concurrent Upload] | Show how to perform upload concurrently to a storage service. |
| [Multipart Upload] | Show how to perform a multipart upload to a storage service. |
[Basic]: ./examples/basic
[Concurrent Upload]: ./examples/concurrent-upload
[Multipart Upload]: ./examples/multipart-upload
## Contributing
Check out the [CONTRIBUTING](./CONTRIBUTING.md) guide for more details on getting started with contributing to this project.
## Used by
Check out the [users](./users.md) list for more details on who is using OpenDAL.
## Branding
The first and most prominent mentions must use the full form: **Apache OpenDAL™** of the name for any individual usage (webpage, handout, slides, etc.) Depending on the context and writing style, you should use the full form of the name sufficiently often to ensure that readers clearly understand the association of both the OpenDAL project and the OpenDAL software product to the ASF as the parent organization.
For more details, see the [Apache Product Name Usage Guide](https://www.apache.org/foundation/marks/guide).
## License and Trademarks
Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation.
```
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
```markdown
# Apache OpenDAL™: *One Layer, All Storage.*
[](https://github.com/apache/opendal/discussions)
[](https://opendal.apache.org/discord)
[](https://deepwiki.com/apache/opendal)
OpenDAL (`/ˈoʊ.pən.dæl/`, pronounced "OH-puhn-dal") is an Open Data Access Layer that enables seamless interaction with diverse storage services.
OpenDAL's development is guided by its vision of **One Layer, All Storage** and its core principles: **Open Community**, **Solid Foundation**, **Fast Access**, **Object Storage First**, and **Extensible Architecture**. Read the explained vision at [OpenDAL Vision](https://opendal.apache.org/vision).
<img src="https://opendal.apache.org/img/architectural.png" alt="OpenDAL Architectural" width="61.8%" />
## For *ANY* languages
> **Note**: Each binding has its own independent version number, which may differ from the Rust core version. When checking for updates or compatibility, always refer to the specific binding's version rather than the core version.
| Name | Release | Docs | Used By |
| ----------------- | ------------------------------------------------ | --------------------------------------------------------------------------------- | ----------------------------------- |
| [Rust Core] | [![Rust Core Image]][Rust Core Link] | [![Docs Release]][Rust Core Release Docs] [![Docs Dev]][Rust Core Dev Docs] | [![Rust Core Users Image]][Rust Core Users] |
| [C Binding] | - | [![Docs Dev]][C Binding Dev Docs] | [![C Binding Users Image]][C Binding Users] |
| [Cpp Binding] | - | [![Docs Dev]][Cpp Binding Dev Docs] | - |
| [D Binding] | - | - | - |
| [Dart Binding] | - | - | - |
| [Dotnet Binding] | - | - | - |
| [Go Binding] | [![Go Binding Image]][Go Binding Link] | [![Docs Release]][Go Release Docs] | - |
| [Haskell Binding] | - | - | - |
| [Java Binding] | [![Java Binding Image]][Java Binding Link] | [![Docs Release]][Java Binding Release Docs] [![Docs Dev]][Java Binding Dev Docs] | [![Java Binding Users Image]][Java Binding Users] |
| [Lua Binding] | - | - | - |
| [Node.js Binding] | [![Node.js Binding Image]][Node.js Binding Link] | [![Docs Dev]][Node.js Binding Dev Docs] | - |
| [OCaml Binding] | - | - | - |
| [PHP Binding] | - | - | - |
| [Python Binding] | [![Python Binding Image]][Python Binding Link] | [![Docs Dev]][Python Binding Dev Docs] | [![Python Binding Users Image]][Python Binding Users] |
| [Ruby Binding] | - | - | - |
| [Swift Binding] | - | - | - |
| [Zig Binding] | - | - | - |
[Docs Release]: https://img.shields.io/badge/docs-release-blue
[Docs Dev]: https://img.shields.io/badge/docs-dev-blue
[Rust Core]: core/README.md
[Rust Core Image]: https://img.shields.io/crates/v/opendal.svg
[Rust Core Link]: https://crates.io/crates/opendal
[Rust Core Release Docs]: https://docs.rs/opendal
[Rust Core Dev Docs]: https://opendal.apache.org/docs/rust/opendal/
[Rust Core Users Image]: https://github.com/user-attachments/assets/2726c336-8509-491d-92d8-1be2040d5136
[Rust Core Users]: core/users.md
[C Binding]: bindings/c/README.md
[C Binding Dev Docs]: https://opendal.apache.org/docs/c/
[C Binding Users Image]: https://github.com/user-attachments/assets/b1cf4d79-8478-4eac-ae04-0bbe0d6a993d
[C Binding Users]: bindings/c/users.md
[Cpp Binding]: bindings/cpp/README.md
[Cpp Binding Dev Docs]: https://opendal.apache.org/docs/cpp/
[D Binding]: bindings/d/README.md
[Dart Binding]: bindings/dart/README.md
[Dotnet Binding]: bindings/dotnet/README.md
[Go Binding]: bindings/go/README.md
[Go Binding Image]: https://badge.fury.io/go/github.com%2Fapache%2Fopendal%2Fbindings%2Fgo.svg
[Go Binding Link]: https://pkg.go.dev/github.com/apache/opendal/bindings/go
[Go Release Docs]: https://pkg.go.dev/github.com/apache/opendal/bindings/go
[Haskell Binding]: bindings/haskell/README.md
[Java Binding]: bindings/java/README.md
[Java Binding Image]: https://img.shields.io/maven-central/v/org.apache.opendal/opendal-java
[Java Binding Link]: https://central.sonatype.com/artifact/org.apache.opendal/opendal-java
[Java Binding Release Docs]: https://javadoc.io/doc/org.apache.opendal/opendal-java
[Java Binding Dev Docs]: https://opendal.apache.org/docs/java/
[Java Binding Users Image]: https://github.com/user-attachments/assets/f20a59a9-8f23-4919-a165-980ed4e6e0d0
[Java Binding Users]: bindings/java/users.md
[Lua Binding]: bindings/lua/README.md
[Node.js Binding]: bindings/nodejs/README.md
[Node.js Binding Image]: https://img.shields.io/npm/v/opendal
[Node.js Binding Link]: https://www.npmjs.com/package/opendal
[Node.js Binding Dev Docs]: https://opendal.apache.org/docs/nodejs/
[OCaml Binding]: bindings/ocaml/README.md
[PHP Binding]: bindings/php/README.md
[Python Binding]: bindings/python/README.md
[Python Binding Image]: https://img.shields.io/pypi/v/opendal
[Python Binding Link]: https://pypi.org/project/opendal/
[Python Binding Dev Docs]: https://opendal.apache.org/docs/python/
[Python Binding Users Image]: https://github.com/user-attachments/assets/6bba7e5b-cada-4cf2-81e3-09d4e4535dcb
[Python Binding Users]: bindings/python/users.md
[Ruby Binding]: bindings/ruby/README.md
[Swift Binding]: bindings/swift/README.md
[Zig Binding]: bindings/zig/README.md
## For *ANY* methods
| Name | Description | Release |
| ----- | ------------------------------------------------------------------ | ------------------------- |
| [oli] | Access data via Command Line (alternative to s3cmd, s3cli, azcopy) | [![oli image]][oli crate] |
| [ofs] | Access data via POSIX file system API (alternative to s3fs) | [![ofs image]][ofs crate] |
[oli]: https://github.com/apache/opendal-oli
[oli image]: https://img.shields.io/crates/v/oli.svg
[oli crate]: https://crates.io/crates/oli
[ofs]: https://github.com/apache/opendal-ofs
[ofs image]: https://img.shields.io/crates/v/ofs.svg
[ofs crate]: https://crates.io/crates/ofs
## For *ANY* integrations
| Name | Description | Release | Docs |
| ---------------------- | ----------------------------------------------------------------------------- | ------------------------------------------- | --------------------------------------------------------------------------------- |
| [dav-server-opendalfs] | a [dav-server-rs] implementation using opendal. | [![dav-server image]][dav-server crate] | [![Docs Release]][dav-server release docs] [![Docs Dev]][dav-server dev docs] |
| [object_store_opendal] | an [object_store] implementation using opendal. | [![object_store image]][object_store crate] | [![Docs Release]][object_store release docs] [![Docs Dev]][object_store dev docs] |
| [unftp-sbe-opendal] | an [unftp] storage backend implementation using opendal. | [![unftp-sbe image]][unftp-sbe crate] | [![Docs Release]][unftp-sbe release docs] [![Docs Dev]][unftp-sbe dev docs] |
| [parquet_opendal] | Provides [`parquet`](https://crates.io/crates/parquet) efficient IO utilities | [![parquet image]][parquet crate] | [![Docs Release]][parquet release docs] [![Docs Dev]][parquet dev docs] |
[dav-server-opendalfs]: integrations/dav-server/README.md
[dav-server-rs]: https://github.com/messense/dav-server-rs
[dav-server image]: https://img.shields.io/crates/v/dav-server-opendalfs.svg
[dav-server crate]: https://crates.io/crates/dav-server-opendalfs
[dav-server release docs]: https://docs.rs/dav-server-opendalfs/
[dav-server dev docs]: https://opendal.apache.org/docs/dav-server-opendalfs/dav_server_opendalfs/
[object_store_opendal]: integrations/object_store/README.md
[object_store]: https://docs.rs/object_store
[object_store image]: https://img.shields.io/crates/v/object_store_opendal.svg
[object_store crate]: https://crates.io/crates/object_store_opendal
[object_store release docs]: https://docs.rs/object_store_opendal/
[object_store dev docs]: https://opendal.apache.org/docs/object-store-opendal/object_store_opendal/
[unftp-sbe-opendal]: integrations/unftp-sbe/README.md
[unftp]: https://crates.io/crates/unftp
[unftp-sbe image]: https://img.shields.io/crates/v/unftp-sbe-opendal.svg
[unftp-sbe crate]: https://crates.io/crates/unftp-sbe-opendal
[unftp-sbe release docs]: https://docs.rs/unftp-sbe-opendal/
[unftp-sbe dev docs]: https://opendal.apache.org/docs/unftp-sbe-opendal/unftp_sbe_opendal/
[parquet_opendal]: integrations/parquet/README.md
[parquet image]: https://img.shields.io/crates/v/parquet-opendal.svg
[parquet crate]: https://crates.io/crates/parquet-opendal
[parquet release docs]: https://docs.rs/parquet-opendal/
[parquet dev docs]: https://opendal.apache.org/docs/parquet-opendal/parquet_opendal/
## For *ANY* services
| Type | Services |
| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------- |
| Standard Storage Protocols | ftp http [sftp] [webdav] |
| Object Storage Services | [azblob] [cos] [gcs] [obs] [oss] [s3] <br> [b2] [openstack_swift] [upyun] [vercel-blob] |
| File Storage Services | fs [alluxio] [azdls] [azfile] [compfs] <br> [dbfs] [gridfs] [hdfs] [hdfs-native] [ipfs] [webhdfs] |
| Consumer Cloud Storage Service | [aliyun-drive] [gdrive] [onedrive] [dropbox] [koofr] <br> [pcloud] [seafile] [yandex-disk] |
| Key-Value Storage Services | [cacache] [cloudflare-kv] [dashmap] memory [etcd] <br> [foundationdb] [persy] [redis] [rocksdb] [sled] <br> [redb] [tikv] |
| Database Storage Services | [d1] [mongodb] [mysql] [postgresql] [sqlite] [surrealdb] |
| Cache Storage Services | [ghac] [memcached] [mini-moka] [moka] [vercel-artifacts] |
| Git Based Storage Services | [huggingface] |
[sftp]: https://datatracker.ietf.org/doc/html/draft-ietf-secsh-filexfer-02
[webdav]: https://datatracker.ietf.org/doc/html/rfc4918
[azblob]: https://azure.microsoft.com/en-us/services/storage/blobs/
[cos]: https://www.tencentcloud.com/products/cos
[gcs]: https://cloud.google.com/storage
[obs]: https://www.huaweicloud.com/intl/en-us/product/obs.html
[oss]: https://www.aliyun.com/product/oss
[s3]: https://aws.amazon.com/s3/
[b2]: https://www.backblaze.com/
[openstack_swift]: https://docs.openstack.org/swift/latest/
[upyun]: https://www.upyun.com/
[vercel-blob]: https://vercel.com/docs/storage/vercel-blob
[alluxio]: https://docs.alluxio.io/os/user/stable/en/api/REST-API.html
[azdls]: https://azure.microsoft.com/en-us/products/storage/data-lake-storage/
[azfile]: https://learn.microsoft.com/en-us/rest/api/storageservices/file-service-rest-api
[compfs]: https://github.com/compio-rs/compio/
[dbfs]: https://docs.databricks.com/en/dbfs/index.html
[gridfs]: https://www.mongodb.com/docs/manual/core/gridfs/
[hdfs]: https://hadoop.apache.org/docs/r3.3.4/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html
[hdfs-native]: https://github.com/Kimahriman/hdfs-native
[ipfs]: https://ipfs.tech/
[webhdfs]: https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html
[aliyun-drive]: https://www.aliyundrive.com/
[gdrive]: https://www.google.com/drive/
[onedrive]: https://www.microsoft.com/en-us/microsoft-365/onedrive/online-cloud-storage
[dropbox]: https://www.dropbox.com/
[koofr]: https://koofr.eu/
[pcloud]: https://www.pcloud.com/
[seafile]: https://www.seafile.com/
[yandex-disk]: https://360.yandex.com/disk/
[cacache]: https://crates.io/crates/cacache
[cloudflare-kv]: https://developers.cloudflare.com/kv/
[dashmap]: https://github.com/xacrimon/dashmap
[etcd]: https://etcd.io/
[foundationdb]: https://www.foundationdb.org/
[persy]: https://crates.io/crates/persy
[redis]: https://redis.io/
[rocksdb]: http://rocksdb.org/
[sled]: https://crates.io/crates/sled
[redb]: https://crates.io/crates/redb
[tikv]: https://tikv.org/
[d1]: https://developers.cloudflare.com/d1/
[mongodb]: https://www.mongodb.com/
[mysql]: https://www.mysql.com/
[postgresql]: https://www.postgresql.org/
[sqlite]: https://www.sqlite.org/
[surrealdb]: https://surrealdb.com/
[ghac]: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows
[memcached]: https://memcached.org/
[mini-moka]: https://github.com/moka-rs/mini-moka
[moka]: https://github.com/moka-rs/moka
[vercel-artifacts]: https://vercel.com/docs/concepts/monorepos/remote-caching
[huggingface]: https://huggingface.co/
## Examples
The examples are available at [here](./examples/).
## Documentation
The documentation is available at <https://opendal.apache.org>.
## Contribute
OpenDAL is an active open-source project. We are always open to people who want to use it or contribute to it. Here are some ways to go.
- Start with [Contributing Guide](CONTRIBUTING.md).
- Submit [Issues](https://github.com/apache/opendal/issues/new) for bug report or feature requests.
- Start [Discussions](https://github.com/apache/opendal/discussions/new?category=q-a) for questions or ideas.
- Talk to community directly at [Discord](https://opendal.apache.org/discord).
- Report security vulnerabilities to [private mailing list](mailto:[email protected])
## Branding
The first and most prominent mentions must use the full form: **Apache OpenDAL™** of the name for any individual usage (webpage, handout, slides, etc.) Depending on the context and writing style, you should use the full form of the name sufficiently often to ensure that readers clearly understand the association of both the OpenDAL project and the OpenDAL software product to the ASF as the parent organization.
For more details, see the [Apache Product Name Usage Guide](https://www.apache.org/foundation/marks/guide).
## License and Trademarks
Licensed under the Apache License, Version 2.0: <http://www.apache.org/licenses/LICENSE-2.0>
Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation.
```
--------------------------------------------------------------------------------
/CLAUDE.md:
--------------------------------------------------------------------------------
```markdown
AGENTS.md
```
--------------------------------------------------------------------------------
/core/CONTRIBUTING.md:
--------------------------------------------------------------------------------
```markdown
# Contributing
## Get Started
- `cargo check` to analyze the current package and report errors.
- `cargo build` to compile the current package.
- `cargo clippy` to catch common mistakes and improve code.
- `cargo test --features tests` to run unit tests.
- `cargo test` to run unit tests include doc tests.
- `cargo bench` to run benchmark tests.
Useful tips:
- Check/Build/Test/Clippy all code: `cargo <cmd> --tests --benches --examples`
- Test specific function: `cargo test tests::it::services::fs`
## Tests
We have unit tests and behavior tests.
### Unit Tests
Unit tests are placed under `src/tests`, organized by mod.
To run unit tests:
```shell
cargo test --features tests
```
To run unit tests include doc tests:
```shell
cargo test
```
### Behavior Tests
Behavior Tests are used to make sure every service works correctly.
```shell
# Setup env
cp .env.example .env
# Run tests
cargo test
```
Please visit [Behavior Test README](./tests/behavior/README.md) for more details.
## Benchmark
We use Ops Benchmark Tests to measure every operation's performance on the target platform.
```shell
# Setup env
cp .env.example .env
# Run benches
cargo bench
```
Please visit [Ops Benchmark README](./benches/ops/README.md) for more details.
```
--------------------------------------------------------------------------------
/bindings/nodejs/CONTRIBUTING.md:
--------------------------------------------------------------------------------
```markdown
# Contributing
- [Contributing](#contributing)
- [Setup](#setup)
- [Using a dev container environment](#using-a-dev-container-environment)
- [Bring your own toolbox](#bring-your-own-toolbox)
- [Build](#build)
- [Test](#test)
## Setup
Building `nodejs` bindings requires some extra setup.
For small or first-time contributions, we recommend the dev container method. Prefer to do it yourself? That's fine too!
### Using a dev container environment
OpenDAL provides a pre-configured [dev container](https://containers.dev/) that could be used in [GitHub Codespaces](https://github.com/features/codespaces), [VSCode](https://code.visualstudio.com/), [JetBrains](https://www.jetbrains.com/remote-development/gateway/), [JupyterLab](https://jupyterlab.readthedocs.io/en/stable/). Please pick up your favourite runtime environment.
The fastest way is:
[](https://codespaces.new/apache/opendal?quickstart=1&machine=standardLinux32gb)
### Bring your own toolbox
The `nodejs` binding requires `Node.js@16+` to be built. We recommend using the latest TLS version for development.
OpenDAL provides a `.node-version` file that specifies the recommended node versions. You can use any compatible tool to install the correct node version, such as [fnm](https://github.com/Schniz/fnm).
Alternatively, you can manually install the LTS node by following these steps:
For Ubuntu and Debian:
```shell
> curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash - && sudo apt-get install -y nodejs
```
For RHEL, CentOS, CloudLinux, Amazon Linux or Fedora:
```shell
> curl -fsSL https://rpm.nodesource.com/setup_lts.x | sudo bash -
```
Afterward, you will need to enable `corepack` to ensure that `pnpm` has been set up correctly:
```shell
> sudo corepack enable
```
To verify that everything is working properly, run `pnpm --version`:
```shell
> pnpm --version
8.11.0
```
### Using [devbox](https://www.jetify.com/devbox/docs/)
Launch Development Environment
```shell
devbox shell
```
Contains the following tools:
- `node`: `18`
- `pnpm`: `8.14.0`
- `libiconv`: fix nix `ld` missing issue
[Running Scripts](https://www.jetify.com/devbox/docs/guides/scripts/)
- `devbox run format`
- `devbox run test`
- `devbox run build`
- `devbox run dev`
## Build
```bash
# Install dependencies.
> pnpm install
# Build from source.
> pnpm build
# Build from source with debug info.
> pnpm build:debug
```
## Test
We are using our own developed behavior testing framework.
Taking 'service-memory' as an example, you can use the following command to run it.
```bash
> OPENDAL_TEST=memory pnpm test
✓ |opendal| tests/service.test.mjs (8 tests | 2 skipped) 40ms
Test Files 1 passed (1)
Tests 6 passed | 2 skipped (8)
Start at 01:42:07
Duration 233ms (transform 25ms, setup 0ms, collect 56ms, tests 40ms, environment 0ms, prepare 52ms)
```
```
--------------------------------------------------------------------------------
/bindings/python/CONTRIBUTING.md:
--------------------------------------------------------------------------------
```markdown
# Contributing
First off, thank you for considering contributing to Apache OpenDAL! Your help is
greatly appreciated. This guide will provide you with all the information you need to
get started.
---
## Prerequisites
Before you begin, please ensure you have the following tools installed:
- **Git**: For version control. Setup instructions [here](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git).
- **uv**: To manage the python dependencies and environment. See Installation instructions [here](https://docs.astral.sh/uv/getting-started/installation/).
- **cargo**: To manage the rust dependencies and environment. See Installation instructions [here](https://doc.rust-lang.org/cargo/getting-started/installation.html).
- **taplo**: Simple checker and formatter for TOML files. See Installation instructions [here](https://taplo.tamasfe.dev/cli/installation/cargo.html).
- **hawkeye**: Simple license header checker and formatter. See Installation instructions [here](https://github.com/korandoru/hawkeye?tab=readme-ov-file#cargo-install).
- **Just**: A command runner used to simplify our development workflow. See Installation instructions [here](https://just.systems/man/en/packages.html).
---
## Getting Started: Setup
All development commands are managed through a `justfile`. Setting up your environment
is as simple as running one command.
1. **Clone the repository:**
```shell
git clone [email protected]:apache/opendal.git
cd opendal/bindings/python
```
2. **Set up the virtual environment and install dependencies:**
```shell
just setup
```
This command will create a Python virtual environment in the `.venv` directory and
install all necessary development and runtime dependencies using `uv`.
---
## Development Workflow
The `justfile` provides recipes for common development tasks. You can see all available
commands by running `just --list`.
### Building the Project
You can build a development or release version of the library.
- **Install for development:** This builds the package and installs it in the current
virtual environment, allowing you to immediately use it for testing.
```shell
just install-dev
```
- **Build a distributable wheel:** If you need to create a wheel file for distribution.
```shell
# Build a development wheel
just build-dev
# Build an optimized release wheel
just build-release
```
The built wheel will be located in the `dist/` directory.
### Running Tests
We use `pytest` for testing. The `test` recipe runs all tests within the virtual
environment.
```shell
# Run all tests
just test
# Pass arguments directly to pytest
# For example, run tests for the 'fs' operator matching 'test_write'
OPENDAL_TEST=fs OPENDAL_FS_ROOT=/tmp just test -vk test_write
```
### Code Quality: Formatting & Linting
We use `ruff` for Python linting/formatting and `cargo fmt`/`clippy` for Rust. `just`
makes it easy to ensure your code meets our quality standards.
- **Format all code:**
```shell
just fmt
```
- **Check for linting errors:**
```shell
just lint
```
- **Automatically fix what can be fixed:**
```shell
just fix
```
> **Important\!** Before committing your changes, please run the pre-commit checks to
> ensure everything is formatted and linted correctly.
>
> ```shell
> just pre-commit
> ```
### Building Documentation
The API documentation is built using `mkdocs`. This is automatically handled when you
build the project.
```shell
# Building the project also builds the docs
just build-dev
```
### Cleaning the Workspace
If you need to start fresh, the `clean` command will remove all build artifacts, caches,
and the virtual environment.
```shell
just clean
```
This command removes the `.venv`, build artifacts, and Python cache files.
---
Thank you again for your contribution\!
```
--------------------------------------------------------------------------------
/AGENTS.md:
--------------------------------------------------------------------------------
```markdown
# AGENTS.md
This file provides guidance to AI coding agents (e.g., Claude Code, ChatGPT) when working with code in this repository.
While working on OpenDAL, please remember:
- Always use English in code and comments.
- Only add meaningful comments when the code's behavior is difficult to understand.
- Only add meaningful tests when they actually verify internal behaviors; otherwise, don't create them unless requested.
## Build and Development Commands
### Core Rust Development
- All cargo commands must be executed within the `core` directory.
- All changes to `core` must pass the clippy check and behavior tests.
- Don't change public API or behavior tests unless requested.
```bash
# Check code
cargo check
# Build
cargo build
# Run linter for all services.
cargo clippy --all-targets --all-features -- -D warnings
# Run linter for specific services.
cargo clippy --all-targets --features=services-s3 -- -D warnings
# Run tests (requires test features)
cargo test --features tests
# Run behavior tests
OPENDAL_TEST=s3 cargo test --features services-s3,tests behavior
# Run specific test
cargo test tests::it::services::fs
# Format code
cargo fmt
# Check formatting
cargo fmt --check
# Build documentation
cargo doc --lib --no-deps --all-features
# Run benchmarks
cargo bench
```
### Multi-package Operations
```bash
# Run command across all packages
./scripts/workspace.py cargo check
./scripts/workspace.py cargo fmt -- --check
```
### Code Generation and Release
```bash
# Generate bindings code
just generate python
just generate java
# Update version across project
just update-version
# Release process
just release
```
### Testing Setup
```bash
# Copy environment template for tests
cp .env.example .env
# Edit .env with your service credentials
```
## Architecture Overview
OpenDAL is a unified data access layer with a modular architecture:
### Core Structure
- **core/**: Main Rust library implementing the storage abstraction
- `src/services/`: 50+ storage service implementations (S3, Azure, GCS, etc.)
- `src/layers/`: Middleware for cross-cutting concerns (retry, metrics, tracing)
- `src/raw/`: Low-level traits and types for implementing backends
- `src/types/`: Core types and traits
### Service Implementation Pattern
Each service follows a consistent structure:
- `backend.rs`: Main service implementation with `Accessor` trait
- `config.rs`: Configuration and builder pattern
- `core.rs`: Core logic and HTTP client setup
- `writer.rs`, `reader.rs`, `lister.rs`: Operation implementations
- `error.rs`: Service-specific error handling
### Language Bindings
- **bindings/**: Language-specific bindings (Python, Java, Go, Node.js, etc.)
- Each binding has its own build system and contributing guide
- Generated code uses the `dev` crate for consistency
### Applications
- oli: moved to a separate repository (see docs: https://opendal.apache.org/docs/40-apps/oli)
- ofs: moved to a separate repository (see docs: https://opendal.apache.org/docs/40-apps/ofs)
- oay: removed from this repository
### Integrations
- **integrations/**: Ecosystem integrations (FUSE, WebDAV, object_store)
## Key Development Patterns
### Feature Flags
Services are feature-gated to reduce binary size:
```rust
// Enable specific services
[features]
services-s3 = []
services-azblob = []
```
### Layer System
Layers provide composable middleware:
```rust
op.layer(RetryLayer::new())
.layer(MetricsLayer::new())
.layer(LoggingLayer::new())
```
### Async-First Design
All core operations are async with blocking wrappers available:
```rust
// Async API
let data = op.read("path").await?;
// Blocking API
let data = op.blocking().read("path")?;
```
## Commit Message Format
Use conventional commits:
```
feat(services/gcs): Add start-after support for list
fix(services/s3): Ignore prefix if it's empty
docs: Add troubleshooting guide
ci: Update GitHub Actions workflow
refactor(core): Simplify error handling
```
## Testing Approach
- Unit tests in `src/tests/`
- Behavior tests validate service implementations
- Integration tests require service credentials in `.env`
- CI runs tests across multiple platforms and Rust versions
## Common Tasks
### Adding a New Service
1. Create new module in `core/src/services/`
2. Implement `Accessor` trait
3. Add service feature flag in `Cargo.toml`
4. Add service tests in behavior test suite
5. Update documentation
### Debugging Service Issues
1. Enable debug logging: `RUST_LOG=debug`
2. Use tracing layer for detailed operation tracking
3. Check service-specific error types
4. Verify credentials and endpoint configuration
## Important Notes
- Minimum Rust version: 1.85 (MSRV)
- All services implement the same `Accessor` trait
- Use `just` for common development tasks
- Check CI workflows for platform-specific requirements
- Services are tested against real backends (credentials required)
```
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
```markdown
# Contributing
First, thank you for contributing to OpenDAL! The goal of this document is to provide everything you need to start contributing to OpenDAL. The following TOC is sorted progressively, starting with the basics and expanding into more specifics.
- [Contributing](#contributing)
- [Your First Contribution](#your-first-contribution)
- [Workflow](#workflow)
- [Git Branches](#git-branches)
- [GitHub Pull Requests](#github-pull-requests)
- [Title](#title)
- [Reviews \& Approvals](#reviews--approvals)
- [Merge Style](#merge-style)
- [CI](#ci)
- [Setup](#setup)
- [Using a dev container environment](#using-a-dev-container-environment)
- [Bring your own toolbox](#bring-your-own-toolbox)
- [Code of Conduct](#code-of-conduct)
## Your First Contribution
1. Ensure your change has an issue! Find an [existing issue](https://github.com/apache/opendal/issues) or [open a new issue](https://github.com/apache/opendal/issues/new).
1. [Fork the OpenDAL repository](https://github.com/apache/opendal/fork) in your own GitHub account.
1. [Create a new Git branch](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-and-deleting-branches-within-your-repository).
1. Make your changes.
1. [Submit the branch as a pull request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) to the main OpenDAL repo. An OpenDAL team member should comment and/or review your pull request within a few days. Although, depending on the circumstances, it may take longer.
## Workflow
### Git Branches
*All* changes must be made in a branch and submitted as [pull requests](#github-pull-requests). OpenDAL does not adopt any type of branch naming style, but please use something descriptive of your changes.
### GitHub Pull Requests
Once your changes are ready you must submit your branch as a [pull request](https://github.com/apache/opendal/pulls).
#### Title
The pull request title must follow the format outlined in the [conventional commits spec](https://www.conventionalcommits.org). [Conventional commits](https://www.conventionalcommits.org) is a standardized format for commit messages. OpenDAL only requires this format for commits on the `main` branch. And because OpenDAL squashes commits before merging branches, this means that only the pull request title must conform to this format.
The following are all good examples of pull request titles:
```text
feat(services/gcs): Add start-after support for list
docs: add hdfs classpath related troubleshoot
ci: Mark job as skipped if owner is not apache
fix(services/s3): Ignore prefix if it's empty
refactor: Polish the implementation of webhdfs
```
#### Reviews & Approvals
All pull requests should be reviewed by at least one OpenDAL committer.
#### Merge Style
All pull requests are squash merged.
We generally discourage large pull requests that are over 300–500 lines of diff.
If you would like to propose a change that is larger, we suggest
coming onto our [Discussions](https://github.com/apache/opendal/discussions) and discussing it with us.
This way we can talk through the solution and discuss if a change that large is even needed!
This will produce a quicker response to the change and likely produce code that aligns better with our process.
### CI
Currently, OpenDAL uses GitHub Actions to run tests. The workflows are defined in `.github/workflows`.
## Setup
For small or first-time contributions, we recommend the dev container method. Prefer to do it yourself? That's fine too!
### Using a dev container environment
OpenDAL provides a pre-configured [dev container](https://containers.dev/)
that could be used in [GitHub Codespaces](https://github.com/features/codespaces),
[VSCode](https://code.visualstudio.com/), [JetBrains](https://www.jetbrains.com/remote-development/gateway/),
[JupyterLab](https://jupyterlab.readthedocs.io/en/stable/).
Please pick up your favourite runtime environment.
The fastest way is:
[](https://codespaces.new/apache/opendal?quickstart=1&machine=standardLinux32gb)
### Bring your own toolbox
OpenDAL is primarily a Rust project. To build OpenDAL, you will need to set up Rust development first. We highly recommend using [rustup](https://rustup.rs/) for the setup process.
For Linux or macOS, use the following command:
```shell
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
```
For Windows, download `rustup-init.exe` from [here](https://win.rustup.rs/x86_64) instead.
Rustup will read OpenDAL's `rust-toolchain.toml` and set up everything else automatically. To ensure that everything works correctly, run `cargo version` under OpenDAL's root directory:
```shell
$ cargo version
cargo 1.85.0 (stable)
```
Some components may require specific setup steps. Please refer to their respective `CONTRIBUTING` documentation for more details.
- [Core](core/CONTRIBUTING.md)
- [Node.js Binding](bindings/nodejs/CONTRIBUTING.md)
- [Python Binding](bindings/python/CONTRIBUTING.md)
- [Java Binding](bindings/java#build)
## Code of Conduct
We expect all community members to follow our [Code of Conduct](https://www.apache.org/foundation/policies/conduct.html).
```
--------------------------------------------------------------------------------
/core/CHANGELOG.md:
--------------------------------------------------------------------------------
```markdown
../CHANGELOG.md
```
--------------------------------------------------------------------------------
/bindings/nodejs/theme/package.json:
--------------------------------------------------------------------------------
```json
{
"type": "module"
}
```
--------------------------------------------------------------------------------
/core/upgrade.md:
--------------------------------------------------------------------------------
```markdown
core/src/docs/upgrade.md
```
--------------------------------------------------------------------------------
/core/DEPENDENCIES.md:
--------------------------------------------------------------------------------
```markdown
# Dependencies
Refer to [DEPENDENCIES.rust.tsv](DEPENDENCIES.rust.tsv) for full list.
```
--------------------------------------------------------------------------------
/bindings/python/pyrightconfig.json:
--------------------------------------------------------------------------------
```json
{
"include": ["python/opendal/"],
"ignore": [
"python/opendal/*.pyi"
],
"reportIgnoreCommentWithoutRule": false
}
```
--------------------------------------------------------------------------------
/bindings/java/users.md:
--------------------------------------------------------------------------------
```markdown
# Users
- [CrateDB](https://github.com/crate/crate): An open-source, distributed SQL database for Real-Time Analytics and Hybrid Search
```
--------------------------------------------------------------------------------
/bindings/python/docs/api/operator.md:
--------------------------------------------------------------------------------
```markdown
::: opendal.Operator
options:
heading: "opendal.Operator"
heading_level: 2
show_source: false
show_bases: false
```
--------------------------------------------------------------------------------
/bindings/python/docs/api/file.md:
--------------------------------------------------------------------------------
```markdown
::: opendal.file.File
options:
heading: "opendal.file.File"
heading_level: 2
show_source: false
show_bases: false
```
--------------------------------------------------------------------------------
/bindings/python/docs/api/async_operator.md:
--------------------------------------------------------------------------------
```markdown
::: opendal.AsyncOperator
options:
heading: "opendal.AsyncOperator"
heading_level: 2
show_source: false
show_bases: false
```
--------------------------------------------------------------------------------
/bindings/python/docs/api/async_file.md:
--------------------------------------------------------------------------------
```markdown
::: opendal.file.AsyncFile
options:
heading: "opendal.file.AsyncFile"
heading_level: 2
show_source: false
show_bases: false
```
--------------------------------------------------------------------------------
/core/services/ipmfs/src/docs.md:
--------------------------------------------------------------------------------
```markdown
## Capabilities
This service can be used to:
- [x] create_dir
- [x] stat
- [x] read
- [x] write
- [x] delete
- [x] list
- [ ] copy
- [ ] rename
- [ ] presign
```
--------------------------------------------------------------------------------
/bindings/python/docs/api/capability.md:
--------------------------------------------------------------------------------
```markdown
::: opendal.capability.Capability
options:
heading: "opendal.capability.Capability"
heading_level: 2
show_source: false
show_bases: false
```
--------------------------------------------------------------------------------
/bindings/java/DEPENDENCIES.md:
--------------------------------------------------------------------------------
```markdown
# Dependencies
## Rust Part Dependencies
Refer to [DEPENDENCIES.rust.tsv](DEPENDENCIES.rust.tsv) for the full list.
## Java Part Dependencies
No extra runtime dependencies.
```
--------------------------------------------------------------------------------
/bindings/nodejs/DEPENDENCIES.md:
--------------------------------------------------------------------------------
```markdown
# Dependencies
## Rust Part Dependencies
Refer to [DEPENDENCIES.rust.tsv](DEPENDENCIES.rust.tsv) for the full list.
## Nodejs Part Dependencies
No extra runtime dependencies.
```
--------------------------------------------------------------------------------
/bindings/python/DEPENDENCIES.md:
--------------------------------------------------------------------------------
```markdown
# Dependencies
## Rust Part Dependencies
Refer to [DEPENDENCIES.rust.tsv](DEPENDENCIES.rust.tsv) for the full list.
## Python Part Dependencies
No extra runtime dependencies.
```
--------------------------------------------------------------------------------
/core/services/opfs/src/docs.md:
--------------------------------------------------------------------------------
```markdown
## Capabilities
This service can be used to:
- [ ] create_dir
- [ ] stat
- [ ] read
- [ ] write
- [ ] delete
- [ ] list
- [ ] copy
- [ ] rename
- [ ] presign
## Configuration
## Example
```
--------------------------------------------------------------------------------
/fixtures/http/nginx.conf:
--------------------------------------------------------------------------------
```
error_log /tmp/error.log;
pid /tmp/nginx.pid;
events {
worker_connections 1024;
}
http {
server {
listen 0.0.0.0:8080;
server_name localhost;
access_log /tmp/access.log;
root /tmp/static;
location / {
}
}
}
```
--------------------------------------------------------------------------------
/core/edge/s3_read_on_wasm/webdriver.json:
--------------------------------------------------------------------------------
```json
{
"moz:firefoxOptions": {
"prefs": {
"media.navigator.streams.fake": true,
"media.navigator.permission.disabled": true
},
"args": []
},
"goog:chromeOptions": {
"args": [
"--use-fake-device-for-media-stream",
"--use-fake-ui-for-media-stream"
]
}
}
```
--------------------------------------------------------------------------------
/bindings/nodejs/tsconfig.json:
--------------------------------------------------------------------------------
```json
{
"compilerOptions": {
"target": "ES2018",
"strict": true,
"moduleResolution": "node",
"module": "CommonJS",
"noUnusedLocals": true,
"noUnusedParameters": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true
},
"include": ["."],
"exclude": ["node_modules", "benchmark", "theme"]
}
```
--------------------------------------------------------------------------------
/core/services/mini_moka/src/docs.md:
--------------------------------------------------------------------------------
```markdown
## Capabilities
This service can be used to:
- [ ] create_dir
- [x] stat
- [x] read
- [x] write
- [x] delete
- [x] list
- [ ] copy
- [ ] rename
- [ ] presign
## Notes
To better assist you in choosing the right cache for your use case,
Here's a comparison table with [moka](https://github.com/moka-rs/moka#choosing-the-right-cache-for-your-use-case)
```
--------------------------------------------------------------------------------
/bindings/nodejs/tsconfig.theme.json:
--------------------------------------------------------------------------------
```json
{
"compilerOptions": {
"target": "es2020",
"module": "NodeNext",
"moduleResolution": "nodenext",
"forceConsistentCasingInFileNames": true,
"strict": true,
"esModuleInterop": true,
"outDir": "theme/dist",
"jsx": "react",
"jsxFactory": "JSX.createElement",
"jsxFragmentFactory": "JSX.Fragment"
},
"include": ["theme"]
}
```
--------------------------------------------------------------------------------
/DEPENDENCIES.md:
--------------------------------------------------------------------------------
```markdown
# OpenDAL Dependencies
Please visit every package's `DEPENDENCIES.md` for a full list of dependencies.
## Check
To check and generate dependencies list of any package of opendal, please run:
```bash
python3 ./scripts/dependencies.py check
```
## Generate
To update the dependencies list of every package, please run:
```bash
python3 ./scripts/dependencies.py generate
```
```
--------------------------------------------------------------------------------
/bindings/python/users.md:
--------------------------------------------------------------------------------
```markdown
# Users
- [Dify](https://github.com/langgenius/dify): An open-source LLM app development platform.
- [wren-engine](https://github.com/Canner/wren-engine): The semantic engine for LLMs, bringing semantic context to AI agents.
- [FileCodeBox](https://github.com/vastsa/FileCodeBox): File Express Cabinet - Anonymous Passcode Sharing Text, Files, Like Taking Express Delivery for Files
```
--------------------------------------------------------------------------------
/.devcontainer/devcontainer.json:
--------------------------------------------------------------------------------
```json
{
"name": "OpenDAL",
"image": "mcr.microsoft.com/devcontainers/rust:bullseye",
"postCreateCommand": "bash .devcontainer/post_create.sh",
"customizations": {
"vscode": {
"extensions": [
"rust-lang.rust-analyzer"
],
"settings": {
"editor.formatOnSave": true,
"files.exclude": {
"**/LICENSE": true
}
}
}
}
}
```
--------------------------------------------------------------------------------
/core/services/cloudflare-kv/src/docs.md:
--------------------------------------------------------------------------------
```markdown
## Capabilities
This service can be used to:
- [x] create_dir
- [x] stat
- [x] read
- [x] write
- [x] delete
- [x] list
- [ ] copy
- [ ] rename
- [ ] ~~presign~~
## Configuration
- `root`: Set the working directory of `OpenDAL`
- `api_token`: Set the token of cloudflare api
- `account_id`: Set the account identifier of cloudflare
- `namespace_id`: Set the namespace identifier of d1
```
--------------------------------------------------------------------------------
/bindings/nodejs/devbox.json:
--------------------------------------------------------------------------------
```json
{
"$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.11.1/.schema/devbox.schema.json",
"packages": ["nodejs@18", "[email protected]", "libiconv"],
"env": {
"OPENDAL_TEST": "memory"
},
"shell": {
"init_hook": ["pnpm install"],
"scripts": {
"format": "pnpm format && cargo fmt",
"test": "pnpm test",
"build": "pnpm build",
"dev": "pnpm build:debug"
}
}
}
```
--------------------------------------------------------------------------------
/fixtures/webdav/config/nginx/http.conf:
--------------------------------------------------------------------------------
```
# the following configuration is for redirect test
server {
listen 81;
server_name localhost;
access_log /tmp/forward-access.log;
error_log /tmp/forward-error.log;
location / {
client_max_body_size 1024M;
# forward all other requests to port 80
proxy_pass http://127.0.0.1:80;
}
location /health {
access_log off;
add_header 'Content-Type' 'application/json';
return 200 '{"status":"UP"}';
}
}
```
--------------------------------------------------------------------------------
/core/core/src/services/memory/docs.md:
--------------------------------------------------------------------------------
```markdown
## Capabilities
This service can be used to:
- [ ] create_dir
- [x] stat
- [x] read
- [x] write
- [x] delete
- [x] list
- [ ] copy
- [ ] rename
- [ ] presign
## Example
### Via Builder
```rust,no_run
use std::sync::Arc;
use anyhow::Result;
use opendal_core::services::Memory;
use opendal_core::Operator;
#[tokio::main]
async fn main() -> Result<()> {
let mut builder = Memory::default().root("/tmp");
let op: Operator = Operator::new(builder)?.finish();
Ok(())
}
```
```