Retire the tags taxonomy and fix tag hygiene site-wide [Draft]

config/_default/config.toml

@@ -47,7 +47,6 @@ ignoreFiles = ["\\.ipynb$", ".ipynb_checkpoints$", "\\.Rmd$", "\\.Rmarkdown$", "
   # pages under /authors/<slug>/ (see layouts/authors/list.html). The taxonomy
   # was disabled because it collided with the content/authors/ section, which
   # suppressed every profile whose term slug matched its folder name.
-  tags = "/tag/:slug/"
   categories = "/category/:slug/"
   publication_types = "/publication-type/:slug/"
 
@@ -81,7 +80,9 @@ ignoreFiles = ["\\.ipynb$", ".ipynb_checkpoints$", "\\.Rmd$", "\\.Rmarkdown$", "
 
 # Taxonomies.
 [taxonomies]
-  tag = "tags"
+  # `tag = "tags"` taxonomy removed (#307): free text, ~1,700 terms, three
+  # quarters of them used once. `tags` stays in front matter — search, the
+  # resource card filter and `[related]` read it directly, no taxonomy needed.
   category = "categories"
   publication_type = "publication_types"
   # `author = "authors"` taxonomy disabled: it collided with the content/authors/

config/_default/params.toml

@@ -70,15 +70,6 @@ social_profiles = [
   "https://facebook.com/forrtproject",
 ]
 
-# Minimum number of entries a `tags` term needs before its archive page is left
-# open to search engines. Tags come from free text on curated resources, and
-# roughly three quarters of them are used exactly once, so without a floor the
-# site publishes ~1,500 near-empty archives that mostly restate a single card.
-# Terms below the floor are still browsable and still pass link equity — they
-# are marked `noindex, follow` and kept out of the sitemap. Set to 0 to index
-# every term again.
-tag_index_min_pages = 5
-
 ############################
 ## Site Features
 ############################

content/resources/index.md

@@ -3,4 +3,145 @@
 type = "widget_page"
 headless = false  # Homepage is headless, other widget pages are not.
 title = "Resources"
+
+# Redirects for the retired `/tag/<term>/` archives (#307). Only the 136 terms
+# that were still indexable: the rest were `noindex`, so nothing points at them.
+aliases = [
+  "/tag/aging-science/",
+  "/tag/aging/",
+  "/tag/analysis/",
+  "/tag/artificial-intelligence/",
+  "/tag/bayesian-statistics/",
+  "/tag/bias/",
+  "/tag/bibliometrics/",
+  "/tag/blog/",
+  "/tag/book/",
+  "/tag/careers/",
+  "/tag/center-for-open-science/",
+  "/tag/citation-analysis/",
+  "/tag/clinical-trials/",
+  "/tag/code/",
+  "/tag/collaboration/",
+  "/tag/computer-software/",
+  "/tag/conceptual-and-statistical-knowledge/",
+  "/tag/covid-19/",
+  "/tag/credibility/",
+  "/tag/culture/",
+  "/tag/data-management/",
+  "/tag/data-processing/",
+  "/tag/data-sharing/",
+  "/tag/data/",
+  "/tag/diversity/",
+  "/tag/drug-discovery/",
+  "/tag/economics/",
+  "/tag/education/",
+  "/tag/epidemiology/",
+  "/tag/epistemology/",
+  "/tag/equity/",
+  "/tag/ethics/",
+  "/tag/experimental-design/",
+  "/tag/exploratory-research/",
+  "/tag/funders/",
+  "/tag/funding/",
+  "/tag/genomics/",
+  "/tag/git/",
+  "/tag/harking/",
+  "/tag/hypothesis-testing/",
+  "/tag/inclusion/",
+  "/tag/inside-your-classroom/",
+  "/tag/interaction/",
+  "/tag/lesson-bank/",
+  "/tag/librarians/",
+  "/tag/machine-learning/",
+  "/tag/materials/",
+  "/tag/medical-journals/",
+  "/tag/medicine/",
+  "/tag/meta-analysis/",
+  "/tag/metadata/",
+  "/tag/metascience/",
+  "/tag/methodology/",
+  "/tag/neuroscience/",
+  "/tag/open-access-publishing/",
+  "/tag/open-access/",
+  "/tag/open-code/",
+  "/tag/open-data/",
+  "/tag/open-educational-resources/",
+  "/tag/open-research/",
+  "/tag/open-scholarship-guidelines/",
+  "/tag/open-scholarship-tools-and-technologies/",
+  "/tag/open-scholarship/",
+  "/tag/open-science-framework/",
+  "/tag/open-science-practices/",
+  "/tag/open-science/",
+  "/tag/open-source-software/",
+  "/tag/open-source/",
+  "/tag/organizing/",
+  "/tag/osca-awards-2025/",
+  "/tag/osf/",
+  "/tag/p-hacking/",
+  "/tag/peer-review/",
+  "/tag/philosophy-of-science/",
+  "/tag/podcast/",
+  "/tag/policy/",
+  "/tag/pre-analysis-plans/",
+  "/tag/preprints/",
+  "/tag/preregistration/",
+  "/tag/psychology/",
+  "/tag/publication-bias/",
+  "/tag/publication-ethics/",
+  "/tag/publishers/",
+  "/tag/publishing/",
+  "/tag/python/",
+  "/tag/qualitative-methods/",
+  "/tag/qualitative-research/",
+  "/tag/qualitative/",
+  "/tag/questionable-research-practices/",
+  "/tag/r/",
+  "/tag/registered-reports/",
+  "/tag/registration/",
+  "/tag/replicability/",
+  "/tag/replication-crisis/",
+  "/tag/replication/",
+  "/tag/reporting-guidelines/",
+  "/tag/reporting/",
+  "/tag/reproducibility-crisis-and-credibility-revolution/",
+  "/tag/reproducibility-knowledge/",
+  "/tag/reproducibility/",
+  "/tag/reproducible-research/",
+  "/tag/research-administration/",
+  "/tag/research-assessment/",
+  "/tag/research-best-practices/",
+  "/tag/research-culture/",
+  "/tag/research-data-management-tools/",
+  "/tag/research-data-management/",
+  "/tag/research-design/",
+  "/tag/research-funding/",
+  "/tag/research-integrity/",
+  "/tag/research-methods/",
+  "/tag/research-reporting-guidelines/",
+  "/tag/research-transparency/",
+  "/tag/research/",
+  "/tag/researcher-degrees-of-freedom/",
+  "/tag/researchers/",
+  "/tag/science-policy/",
+  "/tag/science/",
+  "/tag/scientific-community/",
+  "/tag/scientific-publishing/",
+  "/tag/scientists/",
+  "/tag/shell/",
+  "/tag/simulation/",
+  "/tag/software/",
+  "/tag/sql/",
+  "/tag/statistical-data/",
+  "/tag/statistics/",
+  "/tag/students/",
+  "/tag/systematic-reviews/",
+  "/tag/teaching/",
+  "/tag/transparency/",
+  "/tag/tutorial/",
+  "/tag/version-control/",
+  "/tag/video/",
+  "/tag/website/",
+  "/tag/workflow-tools/",
+]
 +++

content/resources/resource.py

@@ -20,6 +20,7 @@
 import os
 import re
 import sys
+from collections import Counter
 from pathlib import Path
 
 import pandas as pd
@@ -97,21 +98,120 @@ def wrangle_data(df):
     df.fillna('', inplace=True)
 
 
+# The cell is free text: commas were the documented separator, but semicolons
+# are used just as often and newlines arrive with pasted text. Splitting on all
+# three is what stops a list being stored as the one tag "Replication; Teaching".
+TAG_SEPARATORS = re.compile(r'[,;\n]+')
+
+
+def clean_tag(raw: str) -> str:
+    """One tag, normalised, or '' if the value is not a term at all.
+
+    The '&' in the trim set is for "Diversity, Equity, & Inclusion", which the
+    comma split leaves as a stray "& Inclusion". Anything with no letter or
+    digit is a placeholder: the sheet uses blanks, "-" and "." for "no tags".
+    """
+    tag = re.sub(r'\s+', ' ', raw.replace('\u00a0', ' ')).strip()
+    tag = tag.lstrip('#').strip()
+    tag = tag.strip(' .,;:-&')
+    return tag if any(ch.isalnum() for ch in tag) else ''
+
+
+def split_hashtag_run(cell: str) -> str:
+    """Make "#a #b #c" separable, since a run of hashtags is a list.
+
+    Only applied when the cell opens with '#', so an ordinary term carrying a
+    number ("Issue #42") is left as one tag.
+    """
+    return re.sub(r'\s+#', ',#', cell) if cell.lstrip().startswith('#') else cell
+
+
+def split_tags(cell: str) -> list:
+    """A tag cell as a list of clean tags, de-duplicated case-insensitively.
+
+    Case-folded because the same term is submitted in several casings; the
+    first spelling wins here and `canonical_tags` settles it sheet-wide.
+    """
+    tags, seen = [], set()
+    for tag in (clean_tag(part) for part in TAG_SEPARATORS.split(split_hashtag_run(cell))):
+        if tag and tag.casefold() not in seen:
+            seen.add(tag.casefold())
+            tags.append(tag)
+    return tags
+
+
+# Longest all-caps spelling still read as an acronym. Six keeps the sheet's
+# acronyms (OSF, FAIR, UNESCO) and drops its shouting (LECTURE, PSYCHOLOGY).
+# Letters only, so "COVID-19" counts as five.
+ACRONYM_MAX_LETTERS = 6
+
+
+def is_shouted(spelling: str) -> bool:
+    """Whether an all-caps spelling reads as emphasis rather than an acronym."""
+    return spelling.isupper() and len([c for c in spelling if c.isalpha()]) > ACRONYM_MAX_LETTERS
+
+
+# Lowercase in title case unless they open the term. Only ever used to score
+# competing spellings against each other, never to rewrite a tag.
+FUNCTION_WORDS = frozenset(
+    'a an and as at but by for from in into nor of on or the to via vs with'.split()
+)
+
+
+def title_case_score(spelling: str) -> float:
+    """How closely a spelling follows title case, from 0 to 1.
+
+    Breaks the tie between "Evolution of Science" and "Evolution Of Science".
+    An all-caps word counts as capitalised, so "FAIR Data" is not penalised.
+    """
+    words = spelling.split()
+    matches = 0
+    for position, word in enumerate(words):
+        letters = [c for c in word if c.isalpha()]
+        if not letters:
+            matches += 1
+        elif position and word.casefold().strip('.,:;') in FUNCTION_WORDS:
+            matches += not letters[0].isupper()
+        else:
+            matches += letters[0].isupper()
+    return matches / len(words)
+
+
+def canonical_tags(df):
+    """Settle on one spelling per tag across the whole sheet.
+
+    "Open Science", "Open science" and "OPEN SCIENCE" read as three labels on
+    the cards. A winner is picked from the spellings in use rather than Title
+    Case imposed, so "OSF", "COVID-19" and "RMarkdown" survive; ranked by
+    shouted last, most common, closest to title case, then alphabetically for
+    a reproducible rebuild. Terms submitted one way only are left alone.
+    """
+    spellings = {}
+    for tags in df['tags'].values:
+        for tag in tags:
+            spellings.setdefault(tag.casefold(), Counter())[tag] += 1
+
+    def rank(item):
+        spelling, count = item
+        return (is_shouted(spelling), -count, -title_case_score(spelling), spelling)
+
+    canonical = {key: min(counter.items(), key=rank)[0]
+                 for key, counter in spellings.items()}
+
+    df['tags'] = [[canonical[tag.casefold()] for tag in tags] for tags in df['tags'].values]
+
+
 def split_cells(df):
     df['creators'] = [[y.strip() for y in x.split(',')] for x in df['creators'].values]
     df['primary_user'] = [[y.strip() for y in x.split(',')] for x in df['primary_user'].values]
     df['material_type'] = [[y.strip() for y in x.split(',')] for x in df['material_type'].values]
     df['education_level'] = [[y.strip() for y in x.split(',')] for x in df['education_level'].values]
     df['subject_areas'] = [[y.strip() for y in x.split(',')] for x in df['subject_areas'].values]
     df['FORRT_clusters'] = [[y.strip() for y in x.split(',')] for x in df['FORRT_clusters'].values]
-    # `tags` becomes a Hugo taxonomy, so a blank cell would otherwise yield
-    # [""] and collect every such resource into one archive; the sheet also
-    # carries "-" and "." as stand-ins for "no tags". Anything without a
-    # letter or digit is a placeholder, not a term, so drop it — the resource
-    # itself is kept, it just ends up untagged.
-    df['tags'] = [[y.strip() for y in x.split(',')
-                   if any(ch.isalnum() for ch in y)]
-                  for x in df['tags'].values]
+    # Tags are free text rather than a controlled vocabulary, so unlike the
+    # columns above they need separator, placeholder and duplicate handling. A
+    # resource with no usable tags is kept, just untagged.
+    df['tags'] = [split_tags(x) for x in df['tags'].values]
     df['language'] = [[y.strip() for y in x.split(',')] for x in df['language'].values]
 
 
@@ -170,6 +270,9 @@ def main():
 
     split_cells(FORRT)
 
+    # After splitting, so every tag in the sheet is compared in its final form.
+    canonical_tags(FORRT)
+
     # Create files
 
     f_path = Path.cwd() / 'content' / 'curated_resources'

content/tags/_index.md

@@ -0,0 +1,7 @@
++++
+title = "Tags"
+description = "Every tag used across the website, with the number of pages carrying it."
+
+# JSON carries the page list the browser filters; see layouts/tags/list.json.
+outputs = ["HTML", "JSON"]
++++

layouts/_default/index.json

@@ -78,7 +78,7 @@
       {{- end -}}
     {{- end -}}
 
-    {{- $index = $index | append (dict "objectID" .File.UniqueID "date" .Date.UTC.Unix "publishdate" .PublishDate "lastmod" .Lastmod.UTC.Unix "expirydate" .ExpiryDate.UTC.Unix "lang" .Lang "permalink" $permalink "relpermalink" $rel_permalink "title" $title "summary" (plainify $desc) "content" .Plain "authors" $authors "kind" .Kind "type" .Type "section" .Section "tags" .Params.Tags "categories" .Params.Categories "image" .Params.image) -}}
+    {{- $index = $index | append (dict "objectID" .File.UniqueID "date" .Date.UTC.Unix "publishdate" .PublishDate "lastmod" .Lastmod.UTC.Unix "expirydate" .ExpiryDate.UTC.Unix "lang" .Lang "permalink" $permalink "relpermalink" $rel_permalink "title" $title "summary" (plainify $desc) "content" .Plain "authors" $authors "kind" .Kind "type" .Type "section" .Section "tags" (partial "functions/page_tags.html" .) "categories" .Params.Categories "image" .Params.image) -}}
 
   {{- end -}}
 {{- end -}}

layouts/partials/custom_head.html

@@ -10,13 +10,6 @@
 
 {{ partial "structured-data/breadcrumbs.html" . }}
 
-{{/* Thin `tags` archives: browsable and still passing link equity, but kept
-     out of search results. See `functions/is_thin_taxonomy.html`, which the
-     sitemap template consults too so the two cannot disagree. */}}
-{{ if partial "functions/is_thin_taxonomy.html" . }}
-  <meta name="robots" content="noindex, follow">
-{{ end }}
-
 {{/* Author profile pages: noindex empty template stubs, and emit Person
      structured data for the substantive ones. (Computed once.) */}}
 {{ if and (eq .Section "authors") .Params.name }}

layouts/partials/custom_js.html

@@ -9,3 +9,8 @@
 {{ if $isClustersPage }}
 <script src="{{ "js/featured-resources.js" | relURL }}" defer></script>
 {{ end }}
+
+{{/* Tag browser: /tags/ filters a site-wide list of tagged pages (issue #307). */}}
+{{ if and .File (strings.HasPrefix .File.Path "tags/") }}
+<script src="{{ "js/tag-browser.js" | relURL }}" defer></script>
+{{ end }}