From 6529ff379ac5737513fe92ba0060d94407ccb58d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 23 May 2025 17:02:23 +0200 Subject: [PATCH 01/10] documentation updates --- doc/book/reference-manual/configuration.md | 14 +++++++------- doc/book/reference-manual/s3-compatibility.md | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index 09ce8d24..091419d9 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -93,30 +93,30 @@ The following gives details about each available configuration option. [Environment variables](#env_variables). -Top-level configuration options: +Top-level configuration options, in alphabetical order: +[`allow_punycode`](#allow_punycode), [`allow_world_readable_secrets`](#allow_world_readable_secrets), [`block_ram_buffer_max`](#block_ram_buffer_max), [`block_size`](#block_size), [`bootstrap_peers`](#bootstrap_peers), [`compression_level`](#compression_level), +[`consistency_mode`](#consistency_mode), [`data_dir`](#data_dir), [`data_fsync`](#data_fsync), [`db_engine`](#db_engine), [`disable_scrub`](#disable_scrub), -[`use_local_tz`](#use_local_tz), [`lmdb_map_size`](#lmdb_map_size), [`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval), [`metadata_dir`](#metadata_dir), [`metadata_fsync`](#metadata_fsync), [`metadata_snapshots_dir`](#metadata_snapshots_dir), [`replication_factor`](#replication_factor), -[`consistency_mode`](#consistency_mode), [`rpc_bind_addr`](#rpc_bind_addr), [`rpc_bind_outgoing`](#rpc_bind_outgoing), [`rpc_public_addr`](#rpc_public_addr), [`rpc_public_addr_subnet`](#rpc_public_addr_subnet) -[`rpc_secret`/`rpc_secret_file`](#rpc_secret). -[`allow_punycode`](#allow_punycode). +[`rpc_secret`/`rpc_secret_file`](#rpc_secret), +[`use_local_tz`](#use_local_tz). The `[consul_discovery]` section: [`api`](#consul_api), @@ -171,7 +171,7 @@ values in the configuration file: ### Top-level configuration options -#### `replication_factor` {#replication_factor} +#### `replication_factor` (since `v1.0.0`) {#replication_factor} The replication factor can be any positive integer smaller or equal the node count in your cluster. The chosen replication factor has a big impact on the cluster's failure tolerancy and performance characteristics. @@ -219,7 +219,7 @@ is in progress. In theory, no data should be lost as rebalancing is a routine operation for Garage, although we cannot guarantee you that everything will go right in such an extreme scenario. -#### `consistency_mode` {#consistency_mode} +#### `consistency_mode` (since `v1.0.0`) {#consistency_mode} The consistency mode setting determines the read and write behaviour of your cluster. diff --git a/doc/book/reference-manual/s3-compatibility.md b/doc/book/reference-manual/s3-compatibility.md index d2c47f3e..edf8de0d 100644 --- a/doc/book/reference-manual/s3-compatibility.md +++ b/doc/book/reference-manual/s3-compatibility.md @@ -23,7 +23,6 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the - 2022-05-25 - Many Ceph S3 endpoints are not documented but implemented. Following a notification from the Ceph community, we added them. - ## High-level features | Feature | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) | @@ -34,6 +33,7 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the | [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) | ✅ Implemented | ❌| ✅| ✅ | ✅ | | [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) | ✅ Implemented | ❌| ✅ | ✅ | ✅(❓) | | [SSE-C encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) | ✅ Implemented | ❓ | ✅ | ❌ | ✅ | +| [Bucket versioning](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html) | ❌ Missing | ✅ | ✅ | ❌ | ✅ | *Note:* OpenIO does not says if it supports presigned URLs. Because it is part of signature v4 and they claim they support it without additional precisions, From ffbce0f689a05975a5cd68b312bfefbad2dccf2b Mon Sep 17 00:00:00 2001 From: Yureka Date: Mon, 12 May 2025 19:39:20 +0200 Subject: [PATCH 02/10] speed up UploadPartCopy (cherry picked from commit db54bf96c7e35851ffbcf3f93fcefb0b9da72000) --- src/api/s3/copy.rs | 73 +++++++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index edda7e0f..969541ad 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -559,6 +559,7 @@ pub async fn handle_upload_part_copy( let mut current_offset = 0; let mut next_block = defragmenter.next().await?; + let mut blocks_to_dup = dest_version.clone(); // TODO this could be optimized similarly to read_and_put_blocks // low priority because uploadpartcopy is rarely used @@ -588,8 +589,7 @@ pub async fn handle_upload_part_copy( .unwrap()?; checksummer = checksummer_updated; - dest_version.blocks.clear(); - dest_version.blocks.put( + let (version_block_key, version_block) = ( VersionBlockKey { part_number, offset: current_offset, @@ -601,37 +601,56 @@ pub async fn handle_upload_part_copy( ); current_offset += data_len; - let block_ref = BlockRef { - block: final_hash, - version: dest_version_id, - deleted: false.into(), + let next = if let Some(final_data) = data_to_upload { + dest_version.blocks.clear(); + dest_version.blocks.put(version_block_key, version_block); + let block_ref = BlockRef { + block: final_hash, + version: dest_version_id, + deleted: false.into(), + }; + let (_, _, _, next) = futures::try_join!( + // Thing 1: if the block is not exactly a block that existed before, + // we need to insert that data as a new block. + garage.block_manager.rpc_put_block( + final_hash, + final_data, + dest_encryption.is_encrypted(), + None + ), + // Thing 2: we need to insert the block in the version + garage.version_table.insert(&dest_version), + // Thing 3: we need to add a block reference + garage.block_ref_table.insert(&block_ref), + // Thing 4: we need to read the next block + defragmenter.next(), + )?; + next + } else { + blocks_to_dup.blocks.put(version_block_key, version_block); + defragmenter.next().await? }; - - let (_, _, _, next) = futures::try_join!( - // Thing 1: if the block is not exactly a block that existed before, - // we need to insert that data as a new block. - async { - if let Some(final_data) = data_to_upload { - garage - .block_manager - .rpc_put_block(final_hash, final_data, dest_encryption.is_encrypted(), None) - .await - } else { - Ok(()) - } - }, - // Thing 2: we need to insert the block in the version - garage.version_table.insert(&dest_version), - // Thing 3: we need to add a block reference - garage.block_ref_table.insert(&block_ref), - // Thing 4: we need to read the next block - defragmenter.next(), - )?; next_block = next; } assert_eq!(current_offset, source_range.length); + // Put the duplicated blocks into the version & block_refs tables + let block_refs_to_put = blocks_to_dup + .blocks + .items() + .iter() + .map(|b| BlockRef { + block: b.1.hash, + version: dest_version_id, + deleted: false.into(), + }) + .collect::>(); + futures::try_join!( + garage.version_table.insert(&blocks_to_dup), + garage.block_ref_table.insert_many(&block_refs_to_put[..]), + )?; + let checksums = checksummer.finalize(); let etag = dest_encryption.etag_from_md5(&checksums.md5); let checksum = checksums.extract(dest_object_checksum_algorithm); From 1b042e379eda36b3b435edfe2ab16a465a9eccaf Mon Sep 17 00:00:00 2001 From: Renjaya Raga Zenta Date: Mon, 19 May 2025 18:28:00 +0700 Subject: [PATCH 03/10] api: s3: implement get bucket acl --- src/api/s3/api_server.rs | 1 + src/api/s3/bucket.rs | 60 ++++++++++++++++++++++++++++++- src/api/s3/xml.rs | 77 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 1 deletion(-) diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index e26c2b65..337ddb23 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -226,6 +226,7 @@ impl ApiHandler for S3ApiServer { Endpoint::DeleteBucket {} => handle_delete_bucket(ctx).await, Endpoint::GetBucketLocation {} => handle_get_bucket_location(ctx), Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(), + Endpoint::GetBucketAcl {} => handle_get_bucket_acl(ctx), Endpoint::ListObjects { delimiter, encoding_type, diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs index 26e2fc49..55caa6c8 100644 --- a/src/api/s3/bucket.rs +++ b/src/api/s3/bucket.rs @@ -5,7 +5,7 @@ use hyper::{Request, Response, StatusCode}; use garage_model::bucket_alias_table::*; use garage_model::bucket_table::Bucket; use garage_model::garage::Garage; -use garage_model::key_table::Key; +use garage_model::key_table::{Key, KeyParams}; use garage_model::permission::BucketKeyPerm; use garage_table::util::*; use garage_util::crdt::*; @@ -44,6 +44,55 @@ pub fn handle_get_bucket_versioning() -> Result, Error> { .body(string_body(xml))?) } +pub fn handle_get_bucket_acl(ctx: ReqCtx) -> Result, Error> { + let ReqCtx { + bucket_id, api_key, .. + } = ctx; + let key_p = api_key.params().ok_or_internal_error( + "Key should not be in deleted state at this point (in handle_get_bucket_acl)", + )?; + + let mut grants: Vec = vec![]; + let kp = api_key.bucket_permissions(&bucket_id); + + if kp.allow_owner { + grants.push(s3_xml::Grant { + grantee: create_grantee(&key_p, &api_key), + permission: s3_xml::Value("FULL_CONTROL".to_string()), + }); + } else { + if kp.allow_read { + grants.push(s3_xml::Grant { + grantee: create_grantee(&key_p, &api_key), + permission: s3_xml::Value("READ".to_string()), + }); + grants.push(s3_xml::Grant { + grantee: create_grantee(&key_p, &api_key), + permission: s3_xml::Value("READ_ACP".to_string()), + }); + } + if kp.allow_write { + grants.push(s3_xml::Grant { + grantee: create_grantee(&key_p, &api_key), + permission: s3_xml::Value("WRITE".to_string()), + }); + } + } + + let access_control_policy = s3_xml::AccessControlPolicy { + xmlns: (), + owner: None, + acl: s3_xml::AccessControlList { entries: grants }, + }; + + let xml = s3_xml::to_xml_with_header(&access_control_policy)?; + trace!("xml: {}", xml); + + Ok(Response::builder() + .header("Content-Type", "application/xml") + .body(string_body(xml))?) +} + pub async fn handle_list_buckets( garage: &Garage, api_key: &Key, @@ -311,6 +360,15 @@ fn parse_create_bucket_xml(xml_bytes: &[u8]) -> Option> { Some(ret) } +fn create_grantee(key_params: &KeyParams, api_key: &Key) -> s3_xml::Grantee { + s3_xml::Grantee { + xmlns_xsi: (), + typ: "CanonicalUser".to_string(), + display_name: Some(s3_xml::Value(key_params.name.get().to_string())), + id: Some(s3_xml::Value(api_key.key_id.to_string())), + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/api/s3/xml.rs b/src/api/s3/xml.rs index e8af3ec0..fdb36318 100644 --- a/src/api/s3/xml.rs +++ b/src/api/s3/xml.rs @@ -13,6 +13,10 @@ pub fn xmlns_tag(_v: &(), s: S) -> Result { s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/") } +pub fn xmlns_xsi_tag(_v: &(), s: S) -> Result { + s.serialize_str("http://www.w3.org/2001/XMLSchema-instance") +} + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] pub struct Value(#[serde(rename = "$value")] pub String); @@ -319,6 +323,42 @@ pub struct PostObject { pub etag: Value, } +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Grantee { + #[serde(rename = "xmlns:xsi", serialize_with = "xmlns_xsi_tag")] + pub xmlns_xsi: (), + #[serde(rename = "xsi:type")] + pub typ: String, + #[serde(rename = "DisplayName")] + pub display_name: Option, + #[serde(rename = "ID")] + pub id: Option, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Grant { + #[serde(rename = "Grantee")] + pub grantee: Grantee, + #[serde(rename = "Permission")] + pub permission: Value, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct AccessControlList { + #[serde(rename = "Grant")] + pub entries: Vec, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct AccessControlPolicy { + #[serde(serialize_with = "xmlns_tag")] + pub xmlns: (), + #[serde(rename = "Owner")] + pub owner: Option, + #[serde(rename = "AccessControlList")] + pub acl: AccessControlList, +} + #[cfg(test)] mod tests { use super::*; @@ -427,6 +467,43 @@ mod tests { Ok(()) } + #[test] + fn get_bucket_acl_result() -> Result<(), ApiError> { + let grant = Grant { + grantee: Grantee { + xmlns_xsi: (), + typ: "CanonicalUser".to_string(), + display_name: Some(Value("owner_name".to_string())), + id: Some(Value("qsdfjklm".to_string())), + }, + permission: Value("FULL_CONTROL".to_string()), + }; + + let get_bucket_acl = AccessControlPolicy { + xmlns: (), + owner: None, + acl: AccessControlList { + entries: vec![grant], + }, + }; + assert_eq!( + to_xml_with_header(&get_bucket_acl)?, + "\ +\ + \ + \ + \ + owner_name\ + qsdfjklm\ + \ + FULL_CONTROL\ + \ + \ +" + ); + Ok(()) + } + #[test] fn delete_result() -> Result<(), ApiError> { let delete_result = DeleteResult { From 2a4f729b573f5f174ca31f061b0eed80fe48ff90 Mon Sep 17 00:00:00 2001 From: James O'Claire Date: Wed, 28 May 2025 09:49:50 +0800 Subject: [PATCH 04/10] Minor doc change to clarify why the capacity does not matter and how the zone name is used --- doc/book/quick-start/_index.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/book/quick-start/_index.md b/doc/book/quick-start/_index.md index 2db4211b..ff0c1510 100644 --- a/doc/book/quick-start/_index.md +++ b/doc/book/quick-start/_index.md @@ -182,11 +182,12 @@ ID Hostname Address Tag Zone Capacit ## Creating a cluster layout Creating a cluster layout for a Garage deployment means informing Garage -of the disk space available on each node of the cluster -as well as the zone (e.g. datacenter) each machine is located in. +of the disk space available on each node of the cluster, `-c`, +as well as the name of the zone (e.g. datacenter), `-z`, each machine is located in. -For our test deployment, we are using only one node. The way in which we configure -it does not matter, you can simply write: +For our test deployment, we are have only one node with zone named `dc1` and a +capacity of `1G`, though the capacity is ignored for a single node deployment +and can be changed later when adding new nodes. ```bash garage layout assign -z dc1 -c 1G From fc8fc60f6dae85c70a6350fdcfd560f024656c0e Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Fri, 30 May 2025 16:24:12 +0000 Subject: [PATCH 05/10] emit internal error when we detect race condition (#1053) (fix #1050) i went with a `500`/`InternalError`/`Please try again.` because that is something i've seen AWS S3 report while developing other software, and i'm not convinced all clients would understand a 409 conflict properly (GET don't usually conflict) Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1053 Co-authored-by: trinity-1686a Co-committed-by: trinity-1686a --- src/api/s3/copy.rs | 4 +++- src/api/s3/get.rs | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index 969541ad..47a63c82 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -26,7 +26,7 @@ use garage_api_common::signature::checksum::*; use crate::api_server::{ReqBody, ResBody}; use crate::encryption::EncryptionParams; use crate::error::*; -use crate::get::{full_object_byte_stream, PreconditionHeaders}; +use crate::get::{check_version_not_deleted, full_object_byte_stream, PreconditionHeaders}; use crate::multipart; use crate::put::{extract_metadata_headers, save_stream, ChecksumMode, SaveStreamResult}; use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION; @@ -237,6 +237,7 @@ async fn handle_copy_metaonly( .get(&source_version.uuid, &EmptyKey) .await?; let source_version = source_version.ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&source_version)?; // Write an "uploading" marker in Object table // This holds a reference to the object in the Version table @@ -428,6 +429,7 @@ pub async fn handle_upload_part_copy( .get(&source_object_version.uuid, &EmptyKey) .await? .ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&source_version)?; // We want to reuse blocks from the source version as much as possible. // However, we still need to get the data from these blocks diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index 22076603..888a040a 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -19,12 +19,13 @@ use garage_net::stream::ByteStream; use garage_rpc::rpc_helper::OrderTag; use garage_table::EmptyKey; use garage_util::data::*; -use garage_util::error::OkOrMessage; +use garage_util::error::{Error as UtilError, OkOrMessage}; use garage_model::garage::Garage; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; +use garage_api_common::common_error::CommonError; use garage_api_common::helpers::*; use garage_api_common::signature::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE}; @@ -215,6 +216,7 @@ pub async fn handle_head_without_ctx( .get(&object_version.uuid, &EmptyKey) .await? .ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&version)?; let (part_offset, part_end) = calculate_part_bounds(&version, pn).ok_or(Error::InvalidPart)?; @@ -365,6 +367,21 @@ pub async fn handle_get_without_ctx( } } +pub(crate) fn check_version_not_deleted(version: &Version) -> Result<(), Error> { + if version.deleted.get() { + // the version was deleted between when the object_table was consulted + // and now, this could mean the object was deleted, or overriden. + // Rather than say the key doesn't exist, return a transient error + // to signal the client to try again. + return Err(CommonError::InternalError(UtilError::Message( + "conflict/inconsistency between object and version state, version is deleted" + .to_string(), + )) + .into()); + } + Ok(()) +} + async fn handle_get_full( garage: Arc, version: &ObjectVersion, @@ -431,6 +448,7 @@ pub fn full_object_byte_stream( .ok_or_message("channel closed")?; let version = version_fut.await.unwrap()?.ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&version)?; for (i, (_, vb)) in version.blocks.items().iter().enumerate().skip(1) { let stream_block_i = encryption .get_block(&garage, &vb.hash, Some(order_stream.order(i as u64))) @@ -446,6 +464,14 @@ pub fn full_object_byte_stream( { Ok(()) => (), Err(e) => { + // TODO i think this is a bad idea, we should log + // an error and stop there. If the error happens to + // be exactly the size of what hasn't been streamed + // yet, the client will see the request as a + // success + // instead truncating the output notify the client + // something happened with their download, so that + // they can retry it let _ = tx.send(error_stream_item(e)).await; } } @@ -497,7 +523,7 @@ async fn handle_get_range( .get(&version.uuid, &EmptyKey) .await? .ok_or(Error::NoSuchKey)?; - + check_version_not_deleted(&version)?; let body = body_from_blocks_range(garage, encryption, version.blocks.items(), begin, end); Ok(resp_builder.body(body)?) @@ -548,6 +574,8 @@ async fn handle_get_part( .await? .ok_or(Error::NoSuchKey)?; + check_version_not_deleted(&version)?; + let (begin, end) = calculate_part_bounds(&version, part_number).ok_or(Error::InvalidPart)?; From 8843aa92fa1cc1edbb0cffdeac4b0be644e619d9 Mon Sep 17 00:00:00 2001 From: Renjaya Raga Zenta Date: Mon, 11 Nov 2024 14:58:09 +0700 Subject: [PATCH 06/10] feat: add log to journald feature The systemd-journald is used in most major Linux distros that use systemd. This enables logging using the systemd-journald native protocol, instead of just writing to stderr. --- Cargo.lock | 12 +++++++ Cargo.toml | 1 + doc/book/reference-manual/configuration.md | 4 +++ nix/compile.nix | 1 + src/garage/Cargo.toml | 3 ++ src/garage/main.rs | 37 ++++++++++++++++++++++ 6 files changed, 58 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index e65778cc..8301d8ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1250,6 +1250,7 @@ dependencies = [ "timeago", "tokio", "tracing", + "tracing-journald", "tracing-subscriber", ] @@ -4514,6 +4515,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "tracing-journald" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0b4143302cf1022dac868d521e36e8b27691f72c84b3311750d5188ebba657" +dependencies = [ + "libc", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "tracing-log" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 400c1840..b57f890c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,7 @@ pretty_env_logger = "0.5" structopt = { version = "0.3", default-features = false } syslog-tracing = "0.3" tracing = "0.1" +tracing-journald = "0.3.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } heed = { version = "0.11", default-features = false, features = ["lmdb"] } diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index 091419d9..32dc461b 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -160,6 +160,10 @@ variable, it does not exist in the configuration file: Garage daemon send its logs to `syslog` (using the libc `syslog` function) instead of printing to stderr. +- `GARAGE_LOG_TO_JOURNALD` (since `v2.0.0`): set this to `1` or `true` to make the + Garage daemon send its logs to `journald` (using the native protocol of `systemd-journald`) + instead of printing to stderr. + The following environment variables can be used to override the corresponding values in the configuration file: diff --git a/nix/compile.nix b/nix/compile.nix index 8cd88d01..bbadaa37 100644 --- a/nix/compile.nix +++ b/nix/compile.nix @@ -74,6 +74,7 @@ let "metrics" "telemetry-otlp" "syslog" + "journald" ])); featuresStr = lib.concatStringsSep "," rootFeatures; diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index f03c7331..d2785f06 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -57,6 +57,7 @@ opentelemetry.workspace = true opentelemetry-prometheus = { workspace = true, optional = true } opentelemetry-otlp = { workspace = true, optional = true } syslog-tracing = { workspace = true, optional = true } +tracing-journald = { workspace = true, optional = true } [dev-dependencies] garage_api_common.workspace = true @@ -101,6 +102,8 @@ metrics = [ "garage_api_admin/metrics", "opentelemetry-prometheus" ] telemetry-otlp = [ "opentelemetry-otlp" ] # Logging to syslog syslog = [ "syslog-tracing" ] +# Logging to journald +journald = [ "tracing-journald" ] # NOTE: bundled-libs and system-libs should be treat as mutually exclusive; # exactly one of them should be enabled. diff --git a/src/garage/main.rs b/src/garage/main.rs index ac95e854..2703bedd 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -208,6 +208,43 @@ fn init_logging(opt: &Opt) { } } + if std::env::var("GARAGE_LOG_TO_JOURNALD") + .map(|x| x == "1" || x == "true") + .unwrap_or(false) + { + #[cfg(feature = "journald")] + { + use tracing_journald::{Priority, PriorityMappings}; + use tracing_subscriber::layer::SubscriberExt; + use tracing_subscriber::util::SubscriberInitExt; + + let registry = tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().with_writer(std::io::sink)) + .with(env_filter); + match tracing_journald::layer() { + Ok(layer) => { + registry + .with(layer.with_priority_mappings(PriorityMappings { + info: Priority::Informational, + debug: Priority::Debug, + ..PriorityMappings::new() + })) + .init(); + } + Err(e) => { + eprintln!("Couldn't connect to journald: {}.", e); + std::process::exit(1); + } + } + return; + } + #[cfg(not(feature = "journald"))] + { + eprintln!("Journald support is not enabled in this build."); + std::process::exit(1); + } + } + tracing_subscriber::fmt() .with_writer(std::io::stderr) .with_env_filter(env_filter) From 47143b88ad648635d085893977d750ef3a211087 Mon Sep 17 00:00:00 2001 From: eddster2309 Date: Tue, 3 Jun 2025 09:15:57 +0000 Subject: [PATCH 07/10] Add eddster2309/ansible-role-garage as deployment option --- doc/book/cookbook/ansible.md | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/doc/book/cookbook/ansible.md b/doc/book/cookbook/ansible.md index 6d624c9c..2d0a4a83 100644 --- a/doc/book/cookbook/ansible.md +++ b/doc/book/cookbook/ansible.md @@ -8,18 +8,18 @@ have published Ansible roles. We list them and compare them below. ## Comparison of Ansible roles -| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | -|------------------------------------|---------------------------------------------|---------------------------------------------------------------| -| **Runtime** | Systemd | Docker | -| **Target OS** | Any Linux | Any Linux | -| **Architecture** | amd64, arm64, i686 | amd64, arm64 | -| **Additional software** | None | Traefik | -| **Automatic node connection** | ❌ | ✅ | -| **Layout management** | ❌ | ✅ | -| **Manage buckets & keys** | ❌ | ✅ (basic) | -| **Allow custom Garage config** | ✅ | ❌ | -| **Facilitate Garage upgrades** | ✅ | ❌ | -| **Multiple instances on one host** | ✅ | ✅ | +| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster ansible-role-garage](#eddster-ansible-role-garage) | +|------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------| +| **Runtime** | Systemd | Docker | Systemd | +| **Target OS** | Any Linux | Any Linux | Any Linux | +| **Architecture** | amd64, arm64, i686 | amd64, arm64 | amd64 | +| **Additional software** | None | Traefik | Ngnix and Keepalived (optional) | +| **Automatic node connection** | ❌ | ✅ | ✅ | +| **Layout management** | ❌ | ✅ | ✅ | +| **Manage buckets & keys** | ❌ | ✅ (basic) | ✅ | +| **Allow custom Garage config** | ✅ | ❌ | ❌ | +| **Facilitate Garage upgrades** | ✅ | ❌ | ✅ | +| **Multiple instances on one host** | ✅ | ✅ | ❌ | ## zorun/ansible-role-garage @@ -49,3 +49,15 @@ structured DNS names, etc). As a result, this role makes it easier to start with Garage on Ansible, but is less flexible. + +## eddster2309/ansible-role-garage + +[Source code](https://github.com/eddster2309/ansible-role-garage), [Ansible galaxy](https://galaxy.ansible.com/ui/standalone/roles/eddster2309/garage/) + +This role is a opinionated but customisable role using the official Garage +static binaries and only requires Systemd. As such it should work on any +Linux based host. It includes all the nesscary configuration to +automatically setup a clustered Garage deployment. Most Garage +configuration options are exposed through Ansible variables so while you +can't provide a custom config you can get very close. It can optionally +installed a HA nginx deployment with Keepalived. From adfa44ad70b53614b863de7e94d78f54bb4e10c9 Mon Sep 17 00:00:00 2001 From: eddster2309 Date: Tue, 3 Jun 2025 09:22:43 +0000 Subject: [PATCH 08/10] Add architecture support --- doc/book/cookbook/ansible.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/book/cookbook/ansible.md b/doc/book/cookbook/ansible.md index 2d0a4a83..8b0d2969 100644 --- a/doc/book/cookbook/ansible.md +++ b/doc/book/cookbook/ansible.md @@ -12,7 +12,7 @@ have published Ansible roles. We list them and compare them below. |------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------| | **Runtime** | Systemd | Docker | Systemd | | **Target OS** | Any Linux | Any Linux | Any Linux | -| **Architecture** | amd64, arm64, i686 | amd64, arm64 | amd64 | +| **Architecture** | amd64, arm64, i686 | amd64, arm64 | arm64, arm, 386, amd64 | | **Additional software** | None | Traefik | Ngnix and Keepalived (optional) | | **Automatic node connection** | ❌ | ✅ | ✅ | | **Layout management** | ❌ | ✅ | ✅ | From 26bc8079050a52be9a12d02fc49fe20645660c78 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Tue, 10 Jun 2025 20:42:59 +0200 Subject: [PATCH 09/10] put web error in a basic webpage before, it was a plain string, with an xml content type this caused browsers to show very ugly and meaningless pages --- src/garage/tests/s3/website.rs | 42 ++++++++++++++++++++++++++++++++++ src/web/web_server.rs | 24 +++++++++++++++++-- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/src/garage/tests/s3/website.rs b/src/garage/tests/s3/website.rs index 6d37eee8..bbac3de5 100644 --- a/src/garage/tests/s3/website.rs +++ b/src/garage/tests/s3/website.rs @@ -606,3 +606,45 @@ async fn test_website_puny() { ); } } + +#[tokio::test] +async fn test_website_object_not_found() { + const BCKT_NAME: &str = "not-found"; + let ctx = common::context(); + let _bucket = ctx.create_bucket(BCKT_NAME); + + let client = Client::builder(TokioExecutor::new()).build_http(); + + let req = |suffix| { + Request::builder() + .method("GET") + .uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port)) + .header("Host", format!("{}{}", BCKT_NAME, suffix)) + .body(Body::new(Bytes::new())) + .unwrap() + }; + + ctx.garage + .command() + .args(["bucket", "website", "--allow", BCKT_NAME]) + .quiet() + .expect_success_status("Could not allow website on bucket"); + + let resp = client.request(req("")).await.unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + // the error we return by default are *not* xml + assert_eq!( + resp.headers().get(http::header::CONTENT_TYPE).unwrap(), + "text/html; charset=utf-8" + ); + let result = String::from_utf8( + resp.into_body() + .collect() + .await + .unwrap() + .to_bytes() + .to_vec(), + ) + .unwrap(); + assert!(result.contains("not found")); +} diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 242f7801..ea02ab0f 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -397,10 +397,30 @@ fn error_to_res(e: Error) -> Response> { // was a HEAD request or we couldn't get the error document) // We do NOT enter this code path when returning the bucket's // error document (this is handled in serve_file) - let body = string_body(format!("{}\n", e)); - let mut http_error = Response::new(body); + let mut body_str = format!( + r"{http_code} {code_text} +

{http_code} {code_text}

", + http_code = e.http_status_code().as_u16(), + code_text = e.http_status_code().canonical_reason().unwrap_or("Unknown"), + ); + if let Error::ApiError(ref err) = e { + body_str.push_str(&format!( + r" +
    +
  • Code: {s3_code}
    • +
  • Message: {s3_message}.
    • +
", + s3_code = err.aws_code(), + s3_message = err, + )); + } + let mut http_error = Response::new(string_body(body_str)); *http_error.status_mut() = e.http_status_code(); e.add_headers(http_error.headers_mut()); + http_error.headers_mut().insert( + http::header::CONTENT_TYPE, + "text/html; charset=utf-8".parse().unwrap(), + ); http_error } From 85ee4f5d8c10e676e7c9ed4da3734f449ce1ac7c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 13 Jun 2025 13:49:35 +0200 Subject: [PATCH 10/10] cli: mark block refs as deleted in garage block purge --- src/garage/admin/block.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/garage/admin/block.rs b/src/garage/admin/block.rs index edeb88c0..5f908ce4 100644 --- a/src/garage/admin/block.rs +++ b/src/garage/admin/block.rs @@ -101,6 +101,7 @@ impl AdminRpcHandler { let mut obj_dels = 0; let mut mpu_dels = 0; let mut ver_dels = 0; + let mut br_dels = 0; for hash in blocks { let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?; @@ -131,12 +132,19 @@ impl AdminRpcHandler { ver_dels += 1; } } + if !br.deleted.get() { + let mut br = br; + br.deleted.set(); + self.garage.block_ref_table.insert(&br).await?; + br_dels += 1; + } } } Ok(AdminRpc::Ok(format!( - "Purged {} blocks, {} versions, {} objects, {} multipart uploads", + "Purged {} blocks: marked {} block refs, {} versions, {} objects and {} multipart uploads as deleted", blocks.len(), + br_dels, ver_dels, obj_dels, mpu_dels,