Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(memory): Implement shared memory state across Relay #3821

Merged
merged 40 commits into from
Jul 22, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
90fb599
feat(memory): Implement shared memory state across Relay
iambriccardo Jul 15, 2024
b0ca49d
feat(memory): Add a way to track memory usage
iambriccardo Jul 16, 2024
7c8d357
Fix
iambriccardo Jul 16, 2024
cc4b577
Fix
iambriccardo Jul 16, 2024
c449dfd
Fix
iambriccardo Jul 16, 2024
2dc4e13
Fix
iambriccardo Jul 16, 2024
6bfd213
Fix
iambriccardo Jul 16, 2024
584a868
Fix
iambriccardo Jul 16, 2024
e7608a8
Remove usage of floats
iambriccardo Jul 16, 2024
75a3ffc
Improve
iambriccardo Jul 16, 2024
1069847
Improve
iambriccardo Jul 16, 2024
719bea7
Improve
iambriccardo Jul 16, 2024
45b35ee
Improve
iambriccardo Jul 16, 2024
6a8be96
Improve
iambriccardo Jul 17, 2024
f4a7d39
fi
iambriccardo Jul 17, 2024
8ceac2e
Fix
iambriccardo Jul 17, 2024
cc0596d
Fix
iambriccardo Jul 18, 2024
1d3b3e8
Fix
iambriccardo Jul 18, 2024
fa67244
Fix
iambriccardo Jul 18, 2024
08a6e85
Fix
iambriccardo Jul 18, 2024
bd9e159
Fix
iambriccardo Jul 18, 2024
450311d
Fix
iambriccardo Jul 18, 2024
707cb07
Fix
iambriccardo Jul 18, 2024
8670f0d
Fix
iambriccardo Jul 18, 2024
dfc5a9b
Fix
iambriccardo Jul 18, 2024
464ca43
Fix
iambriccardo Jul 18, 2024
e2ca3d4
Merge
iambriccardo Jul 18, 2024
05b1317
Fix
iambriccardo Jul 18, 2024
6823d7d
Update relay-server/src/endpoints/common.rs
iambriccardo Jul 19, 2024
78abada
Update relay-server/src/utils/memory.rs
iambriccardo Jul 19, 2024
8755d87
Update relay-server/src/utils/memory.rs
iambriccardo Jul 19, 2024
f0f2c9f
Update relay-server/src/utils/memory.rs
iambriccardo Jul 19, 2024
aafba46
Update relay-server/src/utils/memory.rs
iambriccardo Jul 19, 2024
998fdf2
Update relay-server/src/utils/memory.rs
iambriccardo Jul 19, 2024
aa1f39e
Update relay-server/src/utils/memory.rs
iambriccardo Jul 19, 2024
735d52c
Improve
iambriccardo Jul 19, 2024
f434135
Merge
iambriccardo Jul 19, 2024
d22dfa6
Merge
iambriccardo Jul 19, 2024
9204a7f
Merge
iambriccardo Jul 19, 2024
8a70bf2
Changelog
iambriccardo Jul 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix
  • Loading branch information
iambriccardo committed Jul 18, 2024
commit 464ca43a82831eaaf8eda3309ba2c2eb26be8138
2 changes: 1 addition & 1 deletion relay-server/src/endpoints/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ pub async fn handle_envelope(
)
}

if !state.memory_stat_config().has_enough_memory() {
if state.memory_checker().check_memory().is_exceeded() {
return Err(BadStoreRequest::QueueFailed);
iambriccardo marked this conversation as resolved.
Show resolved Hide resolved
};

Expand Down
12 changes: 6 additions & 6 deletions relay-server/src/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ use crate::services::relays::{RelayCache, RelayCacheService};
use crate::services::store::StoreService;
use crate::services::test_store::{TestStore, TestStoreService};
use crate::services::upstream::{UpstreamRelay, UpstreamRelayService};
use crate::utils::{MemoryStat, MemoryStatConfig};
use crate::utils::{MemoryChecker, MemoryStat};

/// Indicates the type of failure of the server.
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, thiserror::Error)]
Expand Down Expand Up @@ -84,7 +84,7 @@ pub fn create_runtime(name: &str, threads: usize) -> Runtime {
#[derive(Debug)]
struct StateInner {
config: Arc<Config>,
memory_stat_config: MemoryStatConfig,
memory_checker: MemoryChecker,
registry: Registry,
}

Expand Down Expand Up @@ -238,7 +238,7 @@ impl ServiceState {

let state = StateInner {
config: config.clone(),
memory_stat_config: memory_stat.with_config(config),
memory_checker: memory_stat.with_config(config),
registry,
};

Expand All @@ -252,11 +252,11 @@ impl ServiceState {
&self.inner.config
}

/// Returns a reference to the [`MemoryStatConfig`] which is a [`Config`] aware wrapper on the
/// Returns a reference to the [`MemoryChecker`] which is a [`Config`] aware wrapper on the
/// [`MemoryStat`] which gives utility methods to determine whether memory usage is above
/// thresholds set in the [`Config`].
pub fn memory_stat_config(&self) -> &MemoryStatConfig {
&self.inner.memory_stat_config
pub fn memory_checker(&self) -> &MemoryChecker {
&self.inner.memory_checker
}

/// Returns the address of the [`ProjectCache`] service.
Expand Down
14 changes: 6 additions & 8 deletions relay-server/src/services/health_check.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use crate::services::metrics::{AcceptsMetrics, Aggregator};
use crate::services::project_cache::{ProjectCache, SpoolHealth};
use crate::services::upstream::{IsAuthenticated, UpstreamRelay};
use crate::statsd::RelayTimers;
use crate::utils::MemoryStatConfig;
use crate::utils::{MemoryCheck, MemoryChecker};

/// Checks whether Relay is alive and healthy based on its variant.
#[derive(Clone, Copy, Debug, serde::Deserialize)]
Expand Down Expand Up @@ -83,7 +83,7 @@ impl StatusUpdate {
#[derive(Debug)]
pub struct HealthCheckService {
config: Arc<Config>,
memory_stat_config: MemoryStatConfig,
memory_checker: MemoryChecker,
aggregator: Addr<Aggregator>,
upstream_relay: Addr<UpstreamRelay>,
project_cache: Addr<ProjectCache>,
Expand All @@ -95,23 +95,22 @@ impl HealthCheckService {
/// The service does not run. To run the service, use [`start`](Self::start).
pub fn new(
config: Arc<Config>,
memory_stat_config: MemoryStatConfig,
memory_checker: MemoryChecker,
aggregator: Addr<Aggregator>,
upstream_relay: Addr<UpstreamRelay>,
project_cache: Addr<ProjectCache>,
) -> Self {
Self {
config,
memory_stat_config,
memory_checker,
aggregator,
upstream_relay,
project_cache,
}
}

fn system_memory_probe(&mut self) -> Status {
if !self.memory_stat_config.has_enough_memory_percent() {
let memory = self.memory_stat_config.memory_stat.memory();
if let MemoryCheck::Exceeded(memory) = self.memory_checker.check_memory_percent() {
relay_log::error!(
"Not enough memory, {} / {} ({:.2}% >= {:.2}%)",
memory.used,
Expand All @@ -122,8 +121,7 @@ impl HealthCheckService {
return Status::Unhealthy;
}

if !self.memory_stat_config.has_enough_memory_bytes() {
let memory = self.memory_stat_config.memory_stat.memory();
if let MemoryCheck::Exceeded(memory) = self.memory_checker.check_memory_bytes() {
relay_log::error!(
"Not enough memory, {} / {} ({} >= {})",
memory.used,
Expand Down
49 changes: 23 additions & 26 deletions relay-server/src/services/project_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ use crate::services::upstream::UpstreamRelay;

use crate::statsd::{RelayCounters, RelayGauges, RelayHistograms, RelayTimers};
use crate::utils::{
self, GarbageDisposal, ManagedEnvelope, MemoryStatConfig, RetryBackoff, SleepHandle,
self, GarbageDisposal, ManagedEnvelope, MemoryChecker, RetryBackoff, SleepHandle,
};

/// Requests a refresh of a project state from one of the available sources.
Expand Down Expand Up @@ -555,7 +555,7 @@ impl Services {
#[derive(Debug)]
struct ProjectCacheBroker {
config: Arc<Config>,
memory_stat_config: MemoryStatConfig,
memory_checker: MemoryChecker,
services: Services,
metric_outcomes: MetricOutcomes,
// Need hashbrown because extract_if is not stable in std yet.
Expand Down Expand Up @@ -851,7 +851,7 @@ impl ProjectCacheBroker {
// state or we do not need one.
if project_state.is_some()
&& (sampling_state.is_some() || sampling_key.is_none())
&& self.memory_stat_config.has_enough_memory()
&& self.memory_checker.check_memory().is_below()
&& self.global_config.is_ready()
{
return self.handle_processing(key, context);
Expand Down Expand Up @@ -1096,7 +1096,7 @@ impl ProjectCacheBroker {
#[derive(Debug)]
pub struct ProjectCacheService {
config: Arc<Config>,
memory_stat_config: MemoryStatConfig,
memory_checker: MemoryChecker,
services: Services,
metric_outcomes: MetricOutcomes,
redis: Option<RedisPool>,
Expand All @@ -1106,14 +1106,14 @@ impl ProjectCacheService {
/// Creates a new `ProjectCacheService`.
pub fn new(
config: Arc<Config>,
memory_stat_config: MemoryStatConfig,
memory_checker: MemoryChecker,
services: Services,
metric_outcomes: MetricOutcomes,
redis: Option<RedisPool>,
) -> Self {
Self {
config,
memory_stat_config,
memory_checker,
services,
metric_outcomes,
redis,
Expand All @@ -1127,7 +1127,7 @@ impl Service for ProjectCacheService {
fn spawn_handler(self, mut rx: relay_system::Receiver<Self::Interface>) {
let Self {
config,
memory_stat_config,
memory_checker,
services,
metric_outcomes,
redis,
Expand All @@ -1151,7 +1151,7 @@ impl Service for ProjectCacheService {
test_store,
};
let buffer = match BufferService::create(
memory_stat_config.clone(),
memory_checker.clone(),
buffer_services,
config.clone(),
)
Expand Down Expand Up @@ -1192,7 +1192,7 @@ impl Service for ProjectCacheService {
// fetches via the project source.
let mut broker = ProjectCacheBroker {
config: config.clone(),
memory_stat_config,
memory_checker,
projects: hashbrown::HashMap::new(),
garbage_disposal: GarbageDisposal::new(),
source: ProjectSource::start(
Expand Down Expand Up @@ -1334,27 +1334,24 @@ mod tests {
}))
.unwrap()
.into();
let memory_stat_config = MemoryStat::new().with_config(config.clone());
let memory_checker = MemoryStat::new().with_config(config.clone());
let buffer_services = spooler::Services {
outcome_aggregator: services.outcome_aggregator.clone(),
project_cache: services.project_cache.clone(),
test_store: services.test_store.clone(),
};
let buffer = match BufferService::create(
memory_stat_config.clone(),
buffer_services,
config.clone(),
)
.await
{
Ok(buffer) => buffer.start(),
Err(err) => {
relay_log::error!(error = &err as &dyn Error, "failed to start buffer service");
// NOTE: The process will exit with error if the buffer file could not be
// opened or the migrations could not be run.
std::process::exit(1);
}
};
let buffer =
match BufferService::create(memory_checker.clone(), buffer_services, config.clone())
.await
{
Ok(buffer) => buffer.start(),
Err(err) => {
relay_log::error!(error = &err as &dyn Error, "failed to start buffer service");
// NOTE: The process will exit with error if the buffer file could not be
// opened or the migrations could not be run.
std::process::exit(1);
}
};

let metric_stats = MetricStats::new(
Arc::new(Config::default()),
Expand All @@ -1367,7 +1364,7 @@ mod tests {
(
ProjectCacheBroker {
config: config.clone(),
memory_stat_config,
memory_checker,
projects: hashbrown::HashMap::new(),
garbage_disposal: GarbageDisposal::new(),
source: ProjectSource::start(config, services.upstream_relay.clone(), None),
Expand Down
Loading