Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(memory): Implement shared memory state across Relay #3821

Merged
merged 40 commits into from
Jul 22, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
90fb599
feat(memory): Implement shared memory state across Relay
iambriccardo Jul 15, 2024
b0ca49d
feat(memory): Add a way to track memory usage
iambriccardo Jul 16, 2024
7c8d357
Fix
iambriccardo Jul 16, 2024
cc4b577
Fix
iambriccardo Jul 16, 2024
c449dfd
Fix
iambriccardo Jul 16, 2024
2dc4e13
Fix
iambriccardo Jul 16, 2024
6bfd213
Fix
iambriccardo Jul 16, 2024
584a868
Fix
iambriccardo Jul 16, 2024
e7608a8
Remove usage of floats
iambriccardo Jul 16, 2024
75a3ffc
Improve
iambriccardo Jul 16, 2024
1069847
Improve
iambriccardo Jul 16, 2024
719bea7
Improve
iambriccardo Jul 16, 2024
45b35ee
Improve
iambriccardo Jul 16, 2024
6a8be96
Improve
iambriccardo Jul 17, 2024
f4a7d39
fi
iambriccardo Jul 17, 2024
8ceac2e
Fix
iambriccardo Jul 17, 2024
cc0596d
Fix
iambriccardo Jul 18, 2024
1d3b3e8
Fix
iambriccardo Jul 18, 2024
fa67244
Fix
iambriccardo Jul 18, 2024
08a6e85
Fix
iambriccardo Jul 18, 2024
bd9e159
Fix
iambriccardo Jul 18, 2024
450311d
Fix
iambriccardo Jul 18, 2024
707cb07
Fix
iambriccardo Jul 18, 2024
8670f0d
Fix
iambriccardo Jul 18, 2024
dfc5a9b
Fix
iambriccardo Jul 18, 2024
464ca43
Fix
iambriccardo Jul 18, 2024
e2ca3d4
Merge
iambriccardo Jul 18, 2024
05b1317
Fix
iambriccardo Jul 18, 2024
6823d7d
Update relay-server/src/endpoints/common.rs
iambriccardo Jul 19, 2024
78abada
Update relay-server/src/utils/memory.rs
iambriccardo Jul 19, 2024
8755d87
Update relay-server/src/utils/memory.rs
iambriccardo Jul 19, 2024
f0f2c9f
Update relay-server/src/utils/memory.rs
iambriccardo Jul 19, 2024
aafba46
Update relay-server/src/utils/memory.rs
iambriccardo Jul 19, 2024
998fdf2
Update relay-server/src/utils/memory.rs
iambriccardo Jul 19, 2024
aa1f39e
Update relay-server/src/utils/memory.rs
iambriccardo Jul 19, 2024
735d52c
Improve
iambriccardo Jul 19, 2024
f434135
Merge
iambriccardo Jul 19, 2024
d22dfa6
Merge
iambriccardo Jul 19, 2024
9204a7f
Merge
iambriccardo Jul 19, 2024
8a70bf2
Changelog
iambriccardo Jul 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix
  • Loading branch information
iambriccardo committed Jul 18, 2024
commit cc0596dae94ad2cd8b450304425b6ec7da45252a
2 changes: 1 addition & 1 deletion relay-config/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1349,7 +1349,7 @@ pub struct Health {
pub refresh_interval_ms: u64,
/// Maximum memory watermark in bytes.
///
/// By default there is no absolute limit set and the watermark
/// By default, there is no absolute limit set and the watermark
/// is only controlled by setting [`Self::max_memory_percent`].
pub max_memory_bytes: Option<ByteSize>,
/// Maximum memory watermark as a percentage of maximum system memory.
Expand Down
8 changes: 2 additions & 6 deletions relay-server/src/endpoints/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -296,10 +296,6 @@ fn queue_envelope(
// Split off the envelopes by item type.
let envelopes = ProcessingGroup::split_envelope(*managed_envelope.take_envelope());
for (group, envelope) in envelopes {
if !state.memory_stat().has_enough_memory() {
return Err(BadStoreRequest::QueueFailed);
}

let envelope = ManagedEnvelope::standalone(
envelope,
state.outcome_aggregator().clone(),
Expand Down Expand Up @@ -335,9 +331,9 @@ pub async fn handle_envelope(
)
}

if !state.memory_stat().has_enough_memory() {
if !state.memory_stat_config().has_enough_memory() {
return Err(BadStoreRequest::QueueFailed);
iambriccardo marked this conversation as resolved.
Show resolved Hide resolved
}
};

let mut managed_envelope = ManagedEnvelope::standalone(
envelope,
Expand Down
35 changes: 14 additions & 21 deletions relay-server/src/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ use crate::services::relays::{RelayCache, RelayCacheService};
use crate::services::store::StoreService;
use crate::services::test_store::{TestStore, TestStoreService};
use crate::services::upstream::{UpstreamRelay, UpstreamRelayService};
use crate::utils::{BufferGuard, MemoryStat};
use crate::utils::{BufferGuard, MemoryStat, MemoryStatConfig};

/// Indicates the type of failure of the server.
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, thiserror::Error)]
Expand Down Expand Up @@ -84,8 +84,7 @@ pub fn create_runtime(name: &str, threads: usize) -> Runtime {
#[derive(Debug)]
struct StateInner {
config: Arc<Config>,
buffer_guard: Arc<BufferGuard>,
memory_stat: MemoryStat,
memory_stat_config: MemoryStatConfig,
registry: Registry,
}

Expand All @@ -108,9 +107,9 @@ impl ServiceState {
_ => None,
};

let buffer_guard = Arc::new(BufferGuard::new(config.envelope_buffer_size()));

let memory_stat = MemoryStat::new(config.health_max_memory_watermark_percent());
// We create an instance of `MemoryStat` which can be supplied composed with any arbitrary
// configuration object down the line.
let memory_stat = MemoryStat::new();

// Create an address for the `EnvelopeProcessor`, which can be injected into the
// other services.
Expand Down Expand Up @@ -198,7 +197,7 @@ impl ServiceState {
);
ProjectCacheService::new(
config.clone(),
buffer_guard.clone(),
memory_stat.with_config(config.clone()),
project_cache_services,
metric_outcomes,
redis_pool.clone(),
Expand All @@ -207,10 +206,10 @@ impl ServiceState {

let health_check = HealthCheckService::new(
config.clone(),
memory_stat.with_config(config.clone()),
aggregator.clone(),
upstream_relay.clone(),
project_cache.clone(),
memory_stat.clone(),
)
.start();

Expand Down Expand Up @@ -238,9 +237,8 @@ impl ServiceState {
};

let state = StateInner {
config,
buffer_guard,
memory_stat,
config: config.clone(),
memory_stat_config: memory_stat.with_config(config),
registry,
};

Expand All @@ -254,16 +252,11 @@ impl ServiceState {
&self.inner.config
}

/// Returns a reference to the guard of the envelope buffer.
///
/// This can be used to enter new envelopes into the processing queue and reserve a slot in the
/// buffer. See [`BufferGuard`] for more information.
pub fn buffer_guard(&self) -> &BufferGuard {
&self.inner.buffer_guard
}

pub fn memory_stat(&self) -> &MemoryStat {
&self.inner.memory_stat
/// Returns a reference to the [`MemoryStatConfig`] which is a [`Config`] aware wrapper on the
/// [`MemoryStat`] which gives utility methods to determine whether memory usage is above
/// thresholds set in the [`Config`].
pub fn memory_stat_config(&self) -> &MemoryStatConfig {
&self.inner.memory_stat_config
}

/// Returns the address of the [`ProjectCache`] service.
Expand Down
17 changes: 9 additions & 8 deletions relay-server/src/services/health_check.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use crate::services::metrics::{AcceptsMetrics, Aggregator};
use crate::services::project_cache::{ProjectCache, SpoolHealth};
use crate::services::upstream::{IsAuthenticated, UpstreamRelay};
use crate::statsd::RelayTimers;
use crate::utils::MemoryStat;
use crate::utils::{MemoryStat, MemoryStatConfig};

/// Checks whether Relay is alive and healthy based on its variant.
#[derive(Clone, Copy, Debug, serde::Deserialize)]
Expand Down Expand Up @@ -83,10 +83,10 @@ impl StatusUpdate {
#[derive(Debug)]
pub struct HealthCheckService {
config: Arc<Config>,
memory_stat_config: MemoryStatConfig,
aggregator: Addr<Aggregator>,
upstream_relay: Addr<UpstreamRelay>,
project_cache: Addr<ProjectCache>,
memory_stat: MemoryStat,
}

impl HealthCheckService {
Expand All @@ -95,23 +95,23 @@ impl HealthCheckService {
/// The service does not run. To run the service, use [`start`](Self::start).
pub fn new(
config: Arc<Config>,
memory_stat_config: MemoryStatConfig,
aggregator: Addr<Aggregator>,
upstream_relay: Addr<UpstreamRelay>,
project_cache: Addr<ProjectCache>,
memory_stat: MemoryStat,
) -> Self {
Self {
config,
memory_stat_config,
aggregator,
upstream_relay,
project_cache,
config,
memory_stat,
}
}

fn system_memory_probe(&mut self) -> Status {
let memory = self.memory_stat.memory();
if memory.used_percent() >= self.config.health_max_memory_watermark_percent() {
if !self.memory_stat_config.has_enough_memory_percent() {
let memory = self.memory_stat_config.memory_stat.memory();
iambriccardo marked this conversation as resolved.
Show resolved Hide resolved
relay_log::error!(
"Not enough memory, {} / {} ({:.2}% >= {:.2}%)",
memory.used,
Expand All @@ -122,7 +122,8 @@ impl HealthCheckService {
return Status::Unhealthy;
}

if memory.used > self.config.health_max_memory_watermark_bytes() {
if !self.memory_stat_config.has_enough_memory_bytes() {
let memory = self.memory_stat_config.memory_stat.memory();
relay_log::error!(
"Not enough memory, {} / {} ({} >= {})",
memory.used,
Expand Down
Loading