Filter domain-like garbage out of genres
MusicBrainz community tags sometimes carry review-site domain names (e.g. "musikexpress.de", "rollingstone.de") that leaked into the genre field. Add a shared looks_like_url predicate and apply it at ingestion (is_genre_tag, pick_genre_string) and at report read-time (split_genre_list, top_genre_labels) so both new and already-cached garbage are dropped.
This commit is contained in:
15
src/db.rs
15
src/db.rs
@@ -919,11 +919,26 @@ pub fn is_deprioritised_genre(genre: &str) -> bool {
|
|||||||
canonical_genre_key(genre).split_whitespace().count() < 2
|
canonical_genre_key(genre).split_whitespace().count() < 2
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Return true if a genre label looks like a website domain or URL rather than
|
||||||
|
/// a real genre.
|
||||||
|
///
|
||||||
|
/// MusicBrainz community tags sometimes carry magazine / review-site names
|
||||||
|
/// (e.g. "musikexpress.de", "rollingstone.de", "pitchfork.com") that leak into
|
||||||
|
/// the genre field. Genuine genre labels never contain '.', '/' or ':', so any
|
||||||
|
/// label containing one of those is rejected.
|
||||||
|
pub fn looks_like_url(label: &str) -> bool {
|
||||||
|
label.contains('.') || label.contains('/') || label.contains(':')
|
||||||
|
}
|
||||||
|
|
||||||
/// Split a comma-separated genre string into cleaned labels.
|
/// Split a comma-separated genre string into cleaned labels.
|
||||||
|
///
|
||||||
|
/// Domain-like garbage (see [`looks_like_url`]) is dropped here so that any
|
||||||
|
/// such values already cached in the database never reach the report.
|
||||||
fn split_genre_list(csv: &str) -> Vec<String> {
|
fn split_genre_list(csv: &str) -> Vec<String> {
|
||||||
csv.split(',')
|
csv.split(',')
|
||||||
.map(str::trim)
|
.map(str::trim)
|
||||||
.filter(|s| !s.is_empty())
|
.filter(|s| !s.is_empty())
|
||||||
|
.filter(|s| !looks_like_url(s))
|
||||||
.map(|s| s.to_string())
|
.map(|s| s.to_string())
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -719,11 +719,15 @@ const TAG_BLOCKLIST: &[&str] = &[
|
|||||||
///
|
///
|
||||||
/// Rejects:
|
/// Rejects:
|
||||||
/// - Tags containing digits (years like "2022", encodings like "iso-8859-1")
|
/// - Tags containing digits (years like "2022", encodings like "iso-8859-1")
|
||||||
|
/// - Domain-like tags (review-site names such as "musikexpress.de")
|
||||||
/// - Known non-genre technical / language tags
|
/// - Known non-genre technical / language tags
|
||||||
fn is_genre_tag(tag: &str) -> bool {
|
fn is_genre_tag(tag: &str) -> bool {
|
||||||
if tag.chars().any(|c| c.is_ascii_digit()) {
|
if tag.chars().any(|c| c.is_ascii_digit()) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
if db::looks_like_url(tag) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
let lower = tag.to_ascii_lowercase();
|
let lower = tag.to_ascii_lowercase();
|
||||||
!TAG_BLOCKLIST.contains(&lower.as_str())
|
!TAG_BLOCKLIST.contains(&lower.as_str())
|
||||||
}
|
}
|
||||||
@@ -735,7 +739,11 @@ fn is_genre_tag(tag: &str) -> bool {
|
|||||||
/// 2. Community `tags` (filtered to entries with at least one vote and
|
/// 2. Community `tags` (filtered to entries with at least one vote and
|
||||||
/// passing the genre-tag heuristic to strip technical metadata)
|
/// passing the genre-tag heuristic to strip technical metadata)
|
||||||
fn pick_genre_string(genres: Vec<MbGenre>, tags: Vec<MbTag>) -> Option<String> {
|
fn pick_genre_string(genres: Vec<MbGenre>, tags: Vec<MbTag>) -> Option<String> {
|
||||||
let mut names: Vec<String> = genres.into_iter().map(|g| g.name).collect();
|
let mut names: Vec<String> = genres
|
||||||
|
.into_iter()
|
||||||
|
.map(|g| g.name)
|
||||||
|
.filter(|n| !db::looks_like_url(n))
|
||||||
|
.collect();
|
||||||
if names.is_empty() {
|
if names.is_empty() {
|
||||||
names = tags
|
names = tags
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@@ -1553,6 +1561,43 @@ mod tests {
|
|||||||
assert_eq!(from_tags.as_deref(), Some("electronic"));
|
assert_eq!(from_tags.as_deref(), Some("electronic"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_pick_genre_string_rejects_domain_tags() {
|
||||||
|
// Review-site tags such as "musikexpress.de" must never be picked.
|
||||||
|
let from_tags = pick_genre_string(
|
||||||
|
vec![],
|
||||||
|
vec![
|
||||||
|
MbTag {
|
||||||
|
name: "musikexpress.de".to_string(),
|
||||||
|
count: 3,
|
||||||
|
},
|
||||||
|
MbTag {
|
||||||
|
name: "rollingstone.de".to_string(),
|
||||||
|
count: 2,
|
||||||
|
},
|
||||||
|
MbTag {
|
||||||
|
name: "shoegaze".to_string(),
|
||||||
|
count: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
);
|
||||||
|
assert_eq!(from_tags.as_deref(), Some("shoegaze"));
|
||||||
|
|
||||||
|
// Domains must also be stripped from the curated-genres path.
|
||||||
|
let from_genres = pick_genre_string(
|
||||||
|
vec![
|
||||||
|
MbGenre {
|
||||||
|
name: "pitchfork.com".to_string(),
|
||||||
|
},
|
||||||
|
MbGenre {
|
||||||
|
name: "dream pop".to_string(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
vec![],
|
||||||
|
);
|
||||||
|
assert_eq!(from_genres.as_deref(), Some("dream pop"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_uncached_albums_query() {
|
fn test_uncached_albums_query() {
|
||||||
// Verify the uncached_albums query works correctly.
|
// Verify the uncached_albums query works correctly.
|
||||||
|
|||||||
@@ -1727,6 +1727,7 @@ fn top_genre_labels(genre_csv: &str, max_labels: usize) -> Vec<String> {
|
|||||||
.split(',')
|
.split(',')
|
||||||
.map(str::trim)
|
.map(str::trim)
|
||||||
.filter(|s| !s.is_empty())
|
.filter(|s| !s.is_empty())
|
||||||
|
.filter(|s| !db::looks_like_url(s))
|
||||||
.map(|s| s.to_string())
|
.map(|s| s.to_string())
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user