Filter domain-like garbage out of genres

MusicBrainz community tags sometimes carry review-site domain names
(e.g. "musikexpress.de", "rollingstone.de") that leaked into the genre
field. Add a shared looks_like_url predicate and apply it at ingestion
(is_genre_tag, pick_genre_string) and at report read-time
(split_genre_list, top_genre_labels) so both new and already-cached
garbage are dropped.
This commit is contained in:
2026-06-29 17:21:11 +01:00
parent f6e1a6d4ea
commit 23ef54555e
3 changed files with 62 additions and 1 deletions

View File

@@ -919,11 +919,26 @@ pub fn is_deprioritised_genre(genre: &str) -> bool {
canonical_genre_key(genre).split_whitespace().count() < 2 canonical_genre_key(genre).split_whitespace().count() < 2
} }
/// Return true if a genre label looks like a website domain or URL rather than
/// a real genre.
///
/// MusicBrainz community tags sometimes carry magazine / review-site names
/// (e.g. "musikexpress.de", "rollingstone.de", "pitchfork.com") that leak into
/// the genre field. Genuine genre labels never contain '.', '/' or ':', so any
/// label containing one of those is rejected.
pub fn looks_like_url(label: &str) -> bool {
label.contains('.') || label.contains('/') || label.contains(':')
}
/// Split a comma-separated genre string into cleaned labels. /// Split a comma-separated genre string into cleaned labels.
///
/// Domain-like garbage (see [`looks_like_url`]) is dropped here so that any
/// such values already cached in the database never reach the report.
fn split_genre_list(csv: &str) -> Vec<String> { fn split_genre_list(csv: &str) -> Vec<String> {
csv.split(',') csv.split(',')
.map(str::trim) .map(str::trim)
.filter(|s| !s.is_empty()) .filter(|s| !s.is_empty())
.filter(|s| !looks_like_url(s))
.map(|s| s.to_string()) .map(|s| s.to_string())
.collect() .collect()
} }

View File

@@ -719,11 +719,15 @@ const TAG_BLOCKLIST: &[&str] = &[
/// ///
/// Rejects: /// Rejects:
/// - Tags containing digits (years like "2022", encodings like "iso-8859-1") /// - Tags containing digits (years like "2022", encodings like "iso-8859-1")
/// - Domain-like tags (review-site names such as "musikexpress.de")
/// - Known non-genre technical / language tags /// - Known non-genre technical / language tags
fn is_genre_tag(tag: &str) -> bool { fn is_genre_tag(tag: &str) -> bool {
if tag.chars().any(|c| c.is_ascii_digit()) { if tag.chars().any(|c| c.is_ascii_digit()) {
return false; return false;
} }
if db::looks_like_url(tag) {
return false;
}
let lower = tag.to_ascii_lowercase(); let lower = tag.to_ascii_lowercase();
!TAG_BLOCKLIST.contains(&lower.as_str()) !TAG_BLOCKLIST.contains(&lower.as_str())
} }
@@ -735,7 +739,11 @@ fn is_genre_tag(tag: &str) -> bool {
/// 2. Community `tags` (filtered to entries with at least one vote and /// 2. Community `tags` (filtered to entries with at least one vote and
/// passing the genre-tag heuristic to strip technical metadata) /// passing the genre-tag heuristic to strip technical metadata)
fn pick_genre_string(genres: Vec<MbGenre>, tags: Vec<MbTag>) -> Option<String> { fn pick_genre_string(genres: Vec<MbGenre>, tags: Vec<MbTag>) -> Option<String> {
let mut names: Vec<String> = genres.into_iter().map(|g| g.name).collect(); let mut names: Vec<String> = genres
.into_iter()
.map(|g| g.name)
.filter(|n| !db::looks_like_url(n))
.collect();
if names.is_empty() { if names.is_empty() {
names = tags names = tags
.into_iter() .into_iter()
@@ -1553,6 +1561,43 @@ mod tests {
assert_eq!(from_tags.as_deref(), Some("electronic")); assert_eq!(from_tags.as_deref(), Some("electronic"));
} }
#[test]
fn test_pick_genre_string_rejects_domain_tags() {
// Review-site tags such as "musikexpress.de" must never be picked.
let from_tags = pick_genre_string(
vec![],
vec![
MbTag {
name: "musikexpress.de".to_string(),
count: 3,
},
MbTag {
name: "rollingstone.de".to_string(),
count: 2,
},
MbTag {
name: "shoegaze".to_string(),
count: 1,
},
],
);
assert_eq!(from_tags.as_deref(), Some("shoegaze"));
// Domains must also be stripped from the curated-genres path.
let from_genres = pick_genre_string(
vec![
MbGenre {
name: "pitchfork.com".to_string(),
},
MbGenre {
name: "dream pop".to_string(),
},
],
vec![],
);
assert_eq!(from_genres.as_deref(), Some("dream pop"));
}
#[test] #[test]
fn test_uncached_albums_query() { fn test_uncached_albums_query() {
// Verify the uncached_albums query works correctly. // Verify the uncached_albums query works correctly.

View File

@@ -1727,6 +1727,7 @@ fn top_genre_labels(genre_csv: &str, max_labels: usize) -> Vec<String> {
.split(',') .split(',')
.map(str::trim) .map(str::trim)
.filter(|s| !s.is_empty()) .filter(|s| !s.is_empty())
.filter(|s| !db::looks_like_url(s))
.map(|s| s.to_string()) .map(|s| s.to_string())
.collect(); .collect();