Filter domain-like garbage out of genres
MusicBrainz community tags sometimes carry review-site domain names (e.g. "musikexpress.de", "rollingstone.de") that leaked into the genre field. Add a shared looks_like_url predicate and apply it at ingestion (is_genre_tag, pick_genre_string) and at report read-time (split_genre_list, top_genre_labels) so both new and already-cached garbage are dropped.
This commit is contained in:
15
src/db.rs
15
src/db.rs
@@ -919,11 +919,26 @@ pub fn is_deprioritised_genre(genre: &str) -> bool {
|
||||
canonical_genre_key(genre).split_whitespace().count() < 2
|
||||
}
|
||||
|
||||
/// Return true if a genre label looks like a website domain or URL rather than
|
||||
/// a real genre.
|
||||
///
|
||||
/// MusicBrainz community tags sometimes carry magazine / review-site names
|
||||
/// (e.g. "musikexpress.de", "rollingstone.de", "pitchfork.com") that leak into
|
||||
/// the genre field. Genuine genre labels never contain '.', '/' or ':', so any
|
||||
/// label containing one of those is rejected.
|
||||
pub fn looks_like_url(label: &str) -> bool {
|
||||
label.contains('.') || label.contains('/') || label.contains(':')
|
||||
}
|
||||
|
||||
/// Split a comma-separated genre string into cleaned labels.
|
||||
///
|
||||
/// Domain-like garbage (see [`looks_like_url`]) is dropped here so that any
|
||||
/// such values already cached in the database never reach the report.
|
||||
fn split_genre_list(csv: &str) -> Vec<String> {
|
||||
csv.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.filter(|s| !looks_like_url(s))
|
||||
.map(|s| s.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -719,11 +719,15 @@ const TAG_BLOCKLIST: &[&str] = &[
|
||||
///
|
||||
/// Rejects:
|
||||
/// - Tags containing digits (years like "2022", encodings like "iso-8859-1")
|
||||
/// - Domain-like tags (review-site names such as "musikexpress.de")
|
||||
/// - Known non-genre technical / language tags
|
||||
fn is_genre_tag(tag: &str) -> bool {
|
||||
if tag.chars().any(|c| c.is_ascii_digit()) {
|
||||
return false;
|
||||
}
|
||||
if db::looks_like_url(tag) {
|
||||
return false;
|
||||
}
|
||||
let lower = tag.to_ascii_lowercase();
|
||||
!TAG_BLOCKLIST.contains(&lower.as_str())
|
||||
}
|
||||
@@ -735,7 +739,11 @@ fn is_genre_tag(tag: &str) -> bool {
|
||||
/// 2. Community `tags` (filtered to entries with at least one vote and
|
||||
/// passing the genre-tag heuristic to strip technical metadata)
|
||||
fn pick_genre_string(genres: Vec<MbGenre>, tags: Vec<MbTag>) -> Option<String> {
|
||||
let mut names: Vec<String> = genres.into_iter().map(|g| g.name).collect();
|
||||
let mut names: Vec<String> = genres
|
||||
.into_iter()
|
||||
.map(|g| g.name)
|
||||
.filter(|n| !db::looks_like_url(n))
|
||||
.collect();
|
||||
if names.is_empty() {
|
||||
names = tags
|
||||
.into_iter()
|
||||
@@ -1553,6 +1561,43 @@ mod tests {
|
||||
assert_eq!(from_tags.as_deref(), Some("electronic"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pick_genre_string_rejects_domain_tags() {
|
||||
// Review-site tags such as "musikexpress.de" must never be picked.
|
||||
let from_tags = pick_genre_string(
|
||||
vec![],
|
||||
vec![
|
||||
MbTag {
|
||||
name: "musikexpress.de".to_string(),
|
||||
count: 3,
|
||||
},
|
||||
MbTag {
|
||||
name: "rollingstone.de".to_string(),
|
||||
count: 2,
|
||||
},
|
||||
MbTag {
|
||||
name: "shoegaze".to_string(),
|
||||
count: 1,
|
||||
},
|
||||
],
|
||||
);
|
||||
assert_eq!(from_tags.as_deref(), Some("shoegaze"));
|
||||
|
||||
// Domains must also be stripped from the curated-genres path.
|
||||
let from_genres = pick_genre_string(
|
||||
vec![
|
||||
MbGenre {
|
||||
name: "pitchfork.com".to_string(),
|
||||
},
|
||||
MbGenre {
|
||||
name: "dream pop".to_string(),
|
||||
},
|
||||
],
|
||||
vec![],
|
||||
);
|
||||
assert_eq!(from_genres.as_deref(), Some("dream pop"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_uncached_albums_query() {
|
||||
// Verify the uncached_albums query works correctly.
|
||||
|
||||
@@ -1727,6 +1727,7 @@ fn top_genre_labels(genre_csv: &str, max_labels: usize) -> Vec<String> {
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.filter(|s| !db::looks_like_url(s))
|
||||
.map(|s| s.to_string())
|
||||
.collect();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user