Skip to content

Commit fb9e0af

Browse files
jkurijkuri
committed
feat(metadata): implement metadata cache size tracking and add e2e tests, also display metadata cache size on the dashboard on the frontend
1 parent a25ae03 commit fb9e0af

16 files changed

Lines changed: 984 additions & 31 deletions

File tree

‎migrations/2025-07-20-212619_create_metadata_cache/down.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
-- Drop metadata_cache table
2+
DROP INDEX IF EXISTS idx_metadata_cache_last_accessed;
3+
DROP INDEX IF EXISTS idx_metadata_cache_package_name;
4+
DROP TABLE IF EXISTS metadata_cache;

‎migrations/2025-07-20-212619_create_metadata_cache/up.sql‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
-- Create metadata_cache table to track cached metadata.json files
2+
CREATE TABLE metadata_cache (
3+
id INTEGER PRIMARY KEY NOT NULL,
4+
package_name TEXT NOT NULL UNIQUE,
5+
size_bytes BIGINT NOT NULL,
6+
file_path TEXT NOT NULL,
7+
etag TEXT,
8+
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
9+
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
10+
last_accessed TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
11+
access_count INTEGER NOT NULL DEFAULT 0
12+
);
13+
14+
-- Create index for faster lookups
15+
CREATE INDEX idx_metadata_cache_package_name ON metadata_cache(package_name);
16+
CREATE INDEX idx_metadata_cache_last_accessed ON metadata_cache(last_accessed);

‎src/database/metadata_cache.rs‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
use crate::database::connection::{DbPool, get_connection_with_retry};
2+
use crate::models::metadata_cache::{
3+
MetadataCacheRecord, MetadataCacheStats, NewMetadataCacheRecord, UpdateMetadataCacheRecord,
4+
};
5+
use crate::schema::metadata_cache;
6+
use chrono::Utc;
7+
use diesel::prelude::*;
8+
use diesel::sql_types::BigInt;
9+
use log::{debug, warn};
10+
11+
#[derive(QueryableByName)]
12+
struct SumResult {
13+
#[diesel(sql_type = BigInt)]
14+
total: i64,
15+
}
16+
17+
pub struct MetadataCacheOperations<'a> {
18+
pool: &'a DbPool,
19+
}
20+
21+
impl<'a> MetadataCacheOperations<'a> {
22+
pub fn new(pool: &'a DbPool) -> Self {
23+
Self { pool }
24+
}
25+
26+
/// Get metadata cache entry by package name
27+
pub fn get_metadata_cache_entry(
28+
&self,
29+
package_name: &str,
30+
) -> Result<Option<MetadataCacheRecord>, diesel::result::Error> {
31+
let mut conn = get_connection_with_retry(self.pool).map_err(|e| {
32+
diesel::result::Error::DatabaseError(
33+
diesel::result::DatabaseErrorKind::UnableToSendCommand,
34+
Box::new(e.to_string()),
35+
)
36+
})?;
37+
38+
metadata_cache::table
39+
.filter(metadata_cache::package_name.eq(package_name))
40+
.first::<MetadataCacheRecord>(&mut conn)
41+
.optional()
42+
}
43+
44+
/// Create or update metadata cache entry
45+
pub fn upsert_metadata_cache_entry(
46+
&self,
47+
package_name: &str,
48+
size_bytes: i64,
49+
file_path: &str,
50+
etag: Option<&str>,
51+
) -> Result<MetadataCacheRecord, diesel::result::Error> {
52+
let mut conn = get_connection_with_retry(self.pool).map_err(|e| {
53+
diesel::result::Error::DatabaseError(
54+
diesel::result::DatabaseErrorKind::UnableToSendCommand,
55+
Box::new(e.to_string()),
56+
)
57+
})?;
58+
59+
let now = Utc::now().naive_utc();
60+
61+
// Try to update existing record first
62+
let update_result = diesel::update(
63+
metadata_cache::table.filter(metadata_cache::package_name.eq(package_name)),
64+
)
65+
.set(UpdateMetadataCacheRecord {
66+
size_bytes: Some(size_bytes),
67+
file_path: Some(file_path.to_string()),
68+
etag: etag.map(|s| s.to_string()),
69+
updated_at: Some(now),
70+
last_accessed: Some(now),
71+
access_count: None, // Don't reset access count on update
72+
})
73+
.get_result::<MetadataCacheRecord>(&mut conn);
74+
75+
match update_result {
76+
Ok(record) => {
77+
debug!("Updated metadata cache entry for package: {package_name}");
78+
Ok(record)
79+
}
80+
Err(diesel::result::Error::NotFound) => {
81+
// No record exists, create one
82+
let new_record = NewMetadataCacheRecord {
83+
package_name: package_name.to_string(),
84+
size_bytes,
85+
file_path: file_path.to_string(),
86+
etag: etag.map(|s| s.to_string()),
87+
created_at: now,
88+
updated_at: now,
89+
last_accessed: now,
90+
access_count: 0,
91+
};
92+
93+
let result = diesel::insert_into(metadata_cache::table)
94+
.values(&new_record)
95+
.get_result::<MetadataCacheRecord>(&mut conn);
96+
97+
match result {
98+
Ok(record) => {
99+
debug!("Created metadata cache entry for package: {package_name}");
100+
Ok(record)
101+
}
102+
Err(e) => {
103+
warn!("Failed to create metadata cache entry for {package_name}: {e}");
104+
Err(e)
105+
}
106+
}
107+
}
108+
Err(e) => {
109+
warn!("Failed to update metadata cache entry for {package_name}: {e}");
110+
Err(e)
111+
}
112+
}
113+
}
114+
115+
/// Update access info for metadata cache entry
116+
pub fn update_metadata_access_info(
117+
&self,
118+
package_name: &str,
119+
) -> Result<(), diesel::result::Error> {
120+
let mut conn = get_connection_with_retry(self.pool).map_err(|e| {
121+
diesel::result::Error::DatabaseError(
122+
diesel::result::DatabaseErrorKind::UnableToSendCommand,
123+
Box::new(e.to_string()),
124+
)
125+
})?;
126+
127+
let now = Utc::now().naive_utc();
128+
129+
diesel::update(metadata_cache::table.filter(metadata_cache::package_name.eq(package_name)))
130+
.set((
131+
metadata_cache::last_accessed.eq(now),
132+
metadata_cache::access_count.eq(metadata_cache::access_count + 1),
133+
))
134+
.execute(&mut conn)?;
135+
136+
Ok(())
137+
}
138+
139+
/// Get metadata cache statistics
140+
pub fn get_metadata_cache_stats(&self) -> Result<MetadataCacheStats, diesel::result::Error> {
141+
let mut conn = get_connection_with_retry(self.pool).map_err(|e| {
142+
diesel::result::Error::DatabaseError(
143+
diesel::result::DatabaseErrorKind::UnableToSendCommand,
144+
Box::new(e.to_string()),
145+
)
146+
})?;
147+
148+
// Get count
149+
let total_entries: i64 = metadata_cache::table
150+
.count()
151+
.get_result(&mut conn)
152+
.unwrap_or(0);
153+
154+
// Get sum of sizes - use a raw query to handle the sum properly
155+
let total_size_bytes: i64 =
156+
diesel::sql_query("SELECT COALESCE(SUM(size_bytes), 0) as total FROM metadata_cache")
157+
.get_result::<SumResult>(&mut conn)
158+
.map(|result| result.total)
159+
.unwrap_or(0);
160+
161+
Ok(MetadataCacheStats {
162+
total_entries,
163+
total_size_bytes,
164+
total_size_mb: total_size_bytes as f64 / 1024.0 / 1024.0,
165+
})
166+
}
167+
168+
/// Delete metadata cache entry
169+
pub fn delete_metadata_cache_entry(
170+
&self,
171+
package_name: &str,
172+
) -> Result<usize, diesel::result::Error> {
173+
let mut conn = get_connection_with_retry(self.pool).map_err(|e| {
174+
diesel::result::Error::DatabaseError(
175+
diesel::result::DatabaseErrorKind::UnableToSendCommand,
176+
Box::new(e.to_string()),
177+
)
178+
})?;
179+
180+
diesel::delete(metadata_cache::table.filter(metadata_cache::package_name.eq(package_name)))
181+
.execute(&mut conn)
182+
}
183+
184+
/// Clear all metadata cache entries
185+
pub fn clear_metadata_cache(&self) -> Result<usize, diesel::result::Error> {
186+
let mut conn = get_connection_with_retry(self.pool).map_err(|e| {
187+
diesel::result::Error::DatabaseError(
188+
diesel::result::DatabaseErrorKind::UnableToSendCommand,
189+
Box::new(e.to_string()),
190+
)
191+
})?;
192+
193+
diesel::delete(metadata_cache::table).execute(&mut conn)
194+
}
195+
}

‎src/database/mod.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,14 @@
77
//! - `files`: Package file-related database operations
88
//! - `analytics`: Analytics and statistics operations
99
//! - `cache_stats`: Cache statistics operations
10+
//! - `metadata_cache`: Metadata cache operations
1011
//! - `service`: Main DatabaseService that provides a unified interface
1112
1213
pub mod analytics;
1314
pub mod cache_stats;
1415
pub mod connection;
1516
pub mod files;
17+
pub mod metadata_cache;
1618
pub mod packages;
1719
pub mod service;
1820
pub mod versions;
@@ -25,5 +27,6 @@ pub use service::DatabaseService;
2527
pub use analytics::AnalyticsOperations;
2628
pub use cache_stats::CacheStatsOperations;
2729
pub use files::FileOperations;
30+
pub use metadata_cache::MetadataCacheOperations;
2831
pub use packages::PackageOperations;
2932
pub use versions::VersionOperations;

‎src/database/service.rs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ use super::analytics::AnalyticsOperations;
22
use super::cache_stats::CacheStatsOperations;
33
use super::connection::{DbConnection, DbPool, create_pool, get_connection_with_retry};
44
use super::files::{CompletePackageParams, FileOperations, PackageFileParams};
5+
use super::metadata_cache::MetadataCacheOperations;
56
use super::packages::PackageOperations;
67
use super::versions::VersionOperations;
8+
use crate::models::metadata_cache::{MetadataCacheRecord, MetadataCacheStats};
79
use crate::models::package::*;
810

911
/// Main database service that provides a unified interface to all database operations
@@ -192,4 +194,42 @@ impl DatabaseService {
192194
let ops = CacheStatsOperations::new(&self.pool);
193195
ops.increment_miss_count()
194196
}
197+
198+
// Metadata cache operations
199+
pub fn get_metadata_cache_entry(
200+
&self,
201+
package_name: &str,
202+
) -> Result<Option<MetadataCacheRecord>, diesel::result::Error> {
203+
let ops = MetadataCacheOperations::new(&self.pool);
204+
ops.get_metadata_cache_entry(package_name)
205+
}
206+
207+
pub fn upsert_metadata_cache_entry(
208+
&self,
209+
package_name: &str,
210+
size_bytes: i64,
211+
file_path: &str,
212+
etag: Option<&str>,
213+
) -> Result<MetadataCacheRecord, diesel::result::Error> {
214+
let ops = MetadataCacheOperations::new(&self.pool);
215+
ops.upsert_metadata_cache_entry(package_name, size_bytes, file_path, etag)
216+
}
217+
218+
pub fn update_metadata_access_info(
219+
&self,
220+
package_name: &str,
221+
) -> Result<(), diesel::result::Error> {
222+
let ops = MetadataCacheOperations::new(&self.pool);
223+
ops.update_metadata_access_info(package_name)
224+
}
225+
226+
pub fn get_metadata_cache_stats(&self) -> Result<MetadataCacheStats, diesel::result::Error> {
227+
let ops = MetadataCacheOperations::new(&self.pool);
228+
ops.get_metadata_cache_stats()
229+
}
230+
231+
pub fn clear_metadata_cache(&self) -> Result<usize, diesel::result::Error> {
232+
let ops = MetadataCacheOperations::new(&self.pool);
233+
ops.clear_metadata_cache()
234+
}
195235
}

‎src/models/cache.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ pub struct CacheAnalytics {
2828
pub most_popular_packages: Vec<PopularPackage>,
2929
pub recent_packages: Vec<PackageWithVersions>,
3030
pub cache_hit_rate: f64,
31+
pub metadata_cache_entries: i64,
32+
pub metadata_cache_size_bytes: i64,
33+
pub metadata_cache_size_mb: f64,
3134
}
3235

3336
// Database model for persistent cache stats

‎src/models/metadata_cache.rs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
use crate::schema::metadata_cache;
2+
use chrono::NaiveDateTime;
3+
use diesel::prelude::*;
4+
use rocket::serde::Serialize;
5+
6+
#[derive(Queryable, Selectable, Serialize, Debug, Clone)]
7+
#[diesel(table_name = metadata_cache)]
8+
#[diesel(check_for_backend(diesel::sqlite::Sqlite))]
9+
pub struct MetadataCacheRecord {
10+
pub id: i32,
11+
pub package_name: String,
12+
pub size_bytes: i64,
13+
pub file_path: String,
14+
pub etag: Option<String>,
15+
pub created_at: NaiveDateTime,
16+
pub updated_at: NaiveDateTime,
17+
pub last_accessed: NaiveDateTime,
18+
pub access_count: i32,
19+
}
20+
21+
#[derive(Insertable, Debug)]
22+
#[diesel(table_name = metadata_cache)]
23+
pub struct NewMetadataCacheRecord {
24+
pub package_name: String,
25+
pub size_bytes: i64,
26+
pub file_path: String,
27+
pub etag: Option<String>,
28+
pub created_at: NaiveDateTime,
29+
pub updated_at: NaiveDateTime,
30+
pub last_accessed: NaiveDateTime,
31+
pub access_count: i32,
32+
}
33+
34+
#[derive(AsChangeset, Debug)]
35+
#[diesel(table_name = metadata_cache)]
36+
pub struct UpdateMetadataCacheRecord {
37+
pub size_bytes: Option<i64>,
38+
pub file_path: Option<String>,
39+
pub etag: Option<String>,
40+
pub updated_at: Option<NaiveDateTime>,
41+
pub last_accessed: Option<NaiveDateTime>,
42+
pub access_count: Option<i32>,
43+
}
44+
45+
#[derive(Serialize, Debug)]
46+
pub struct MetadataCacheStats {
47+
pub total_entries: i64,
48+
pub total_size_bytes: i64,
49+
pub total_size_mb: f64,
50+
}

‎src/models/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// Re-export all models from their respective modules
22
pub mod auth;
33
pub mod cache;
4-
4+
pub mod metadata_cache;
55
pub mod npm;
66
pub mod package;
77
pub mod user;

0 commit comments

Comments
 (0)