Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,13 @@ that branch until the first workflow deployment succeeds.
5. Imported author lists include contributor-based context and are deduplicated.
6. Review, adjust, and regenerate metadata files before release.

The importer also considers human author names in recent commit history when
repository metadata and contributor profiles do not include everyone. It scans
up to 100 commits without a token or 1,000 with a token, omitting bot names
and GitHub handles found in contributor profiles or anywhere in the scanned
commits rather than using them as citation names. Verify imported authors before
exporting citation metadata.

## Validation Behavior

OpenCite validates metadata at multiple stages:
Expand Down
2 changes: 1 addition & 1 deletion src/App.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -1431,7 +1431,7 @@ export default function App() {
disabled={isZipping}
>
{isZipping && <span className="button-spinner" aria-hidden="true" />}
{isZipping ? 'Creating ZIP…' : 'Download ZIP (Both Files)'}
{isZipping ? 'Creating ZIP…' : 'Download ZIP (All Files)'}
</button>
</div>
<p className="filename-note">
Expand Down
5 changes: 3 additions & 2 deletions src/services/githubApi.js
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,11 @@ export function buildGithubReleaseListApiUrl(owner, repo, perPage = 1) {
return `${API_BASE}/repos/${owner}/${repo}/releases?per_page=${safePerPage}`;
}

export function buildGithubCommitListApiUrl(owner, repo, defaultBranch = '', perPage = 1) {
export function buildGithubCommitListApiUrl(owner, repo, defaultBranch = '', perPage = 1, page = null) {
const safePerPage = Number.isInteger(perPage) ? Math.min(Math.max(perPage, 1), 100) : 1;
const branchFilter = defaultBranch ? `&sha=${encodeURIComponent(defaultBranch)}` : '';
return `${API_BASE}/repos/${owner}/${repo}/commits?per_page=${safePerPage}${branchFilter}`;
const pageFilter = Number.isInteger(page) && page > 1 ? `&page=${page}` : '';
return `${API_BASE}/repos/${owner}/${repo}/commits?per_page=${safePerPage}${branchFilter}${pageFilter}`;
}

export function buildGithubBranchApiUrl(owner, repo, branch) {
Expand Down
11 changes: 9 additions & 2 deletions src/services/githubImporter.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import {
resolveContributorFallbackLimit,
} from './githubImporterContributors.js';
import { dedupeAuthors } from './githubImporterAuthors.js';
import { fetchCommitAuthors } from './githubImporterCommitAuthors.js';
import { addCitationConsistencyWarnings, mergeMetadata } from './githubImporterMerge.js';
export { addCitationConsistencyWarnings, mergeMetadata };
import {
Expand Down Expand Up @@ -549,7 +550,7 @@ export async function importGithubMetadata(repoUrl, options = {}) {
);
const releaseData = Array.isArray(releaseList) && releaseList.length > 0 ? releaseList[0] : null;
const recentCommitPayload = await fetchOptionalJson(
buildGithubCommitListApiUrl(owner, repo, defaultBranch, 10),
buildGithubCommitListApiUrl(owner, repo, defaultBranch, 100),
buildGithubRequestConfig({
authToken,
source: 'commits',
Expand Down Expand Up @@ -716,10 +717,17 @@ export async function importGithubMetadata(repoUrl, options = {}) {
fetchOptionalJson,
extractOrcidFromGithubProfile,
});
const commitAuthors = await fetchCommitAuthors({
owner, repo, defaultBranch, initialCommits: recentCommitPayload,
knownGithubLogins: contributorResult.githubLogins,
warnings, authToken, cleanString, normalizeAuthor, normalizeAuthors,
addWarning, fetchOptionalJson,
});
const coAuthorAuthors = normalizeAuthors(commitCoAuthorNames.map((name) => normalizeAuthor({ name })));
const contributors = dedupeAuthors([
...coAuthorAuthors,
...contributorResult.fallbackAuthors.filter(Boolean),
...commitAuthors,
]);
const contributorLookupAuthors = dedupeAuthors([
...coAuthorAuthors,
Expand Down Expand Up @@ -815,4 +823,3 @@ export async function importGithubMetadata(repoUrl, options = {}) {

return { metadata, warnings, errors, review, healthScan, comparisons };
}

74 changes: 74 additions & 0 deletions src/services/githubImporterCommitAuthors.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import { buildGithubCommitListApiUrl, buildGithubRequestConfig } from './githubApi.js';
import { dedupeAuthors } from './githubImporterAuthors.js';
import { isAutomatedContributorIdentity } from './githubImporterContributors.js';

const COMMIT_PAGE_SIZE = 100;
const UNAUTHENTICATED_PAGE_LIMIT = 1;
const AUTHENTICATED_PAGE_LIMIT = 10;

function isUsableCommitName(name, login, knownLogins, cleanString) {
const normalized = name.toLowerCase();
if (!name || normalized === String(login ?? '').toLowerCase() || knownLogins.has(normalized)) {
return false;
}
if (
/\d/.test(name)
|| /[_@]/.test(name)
|| /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(name)
|| isAutomatedContributorIdentity(name, cleanString)
|| isAutomatedContributorIdentity(login, cleanString)
) {
return false;
}
if (!/\s/.test(name) && name.includes('-') && !name.split('-').every((segment) => /^[A-Z][a-z]+$/.test(segment))) {
return false;
}
return true;
Comment thread
beanbean9339 marked this conversation as resolved.
Resolved
}

export async function fetchCommitAuthors({
owner, repo, defaultBranch, initialCommits = [], knownGithubLogins = [],
warnings, authToken = '', cleanString, normalizeAuthor, normalizeAuthors,
addWarning, fetchOptionalJson,
}) {
const knownLogins = new Set(knownGithubLogins.map((login) => cleanString(login).toLowerCase()));
const discoveredAuthors = [];
let commits = Array.isArray(initialCommits) ? initialCommits : [];
let page = 1;
const maxPages = authToken ? AUTHENTICATED_PAGE_LIMIT : UNAUTHENTICATED_PAGE_LIMIT;

while (true) {
for (const commit of commits) {
const name = cleanString(commit?.commit?.author?.name ?? '');
const login = cleanString(commit?.author?.login ?? '');
if (login) knownLogins.add(login.toLowerCase());
if (commit?.author?.type && commit.author.type !== 'User') continue;
discoveredAuthors.push({ name, login });
}

if (commits.length < COMMIT_PAGE_SIZE) break;
if (page >= maxPages) {
addWarning(warnings, 'commit-authors', 'commit-author-scan-limited',
`Scanned the first ${page * COMMIT_PAGE_SIZE} commits for contributor author names.`,
{ owner, repo, scannedPages: page, scannedCommits: page * COMMIT_PAGE_SIZE });
break;
}

page += 1;
commits = await fetchOptionalJson(
buildGithubCommitListApiUrl(owner, repo, defaultBranch, COMMIT_PAGE_SIZE, page),
buildGithubRequestConfig({
authToken,
source: 'commit-authors',
label: `commit authors page ${page}`,
onWarning: (source, code, message, details = {}) => addWarning(warnings, source, code, message, details),
}),
) || [];
if (!Array.isArray(commits) || commits.length === 0) break;
}

const authorNames = discoveredAuthors
.filter(({ name, login }) => isUsableCommitName(name, login, knownLogins, cleanString))
.map(({ name }) => normalizeAuthor({ name }));
return dedupeAuthors(normalizeAuthors(authorNames));
}
3 changes: 2 additions & 1 deletion src/services/githubImporterContributors.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ const TOP_CONTRIBUTOR_FALLBACK_LIMIT = 4;
const MAX_CONTRIBUTOR_FALLBACK_LIMIT = 20;
const GITHUB_PAGE_SIZE = 100;

function isAutomatedContributorIdentity(value, cleanString) {
export function isAutomatedContributorIdentity(value, cleanString) {
const text = cleanString(value ?? '').trim();
if (!text) {
return false;
Expand Down Expand Up @@ -375,5 +375,6 @@ export async function fetchContributorAuthors({
return {
fallbackAuthors: dedupeAuthors(normalizeAuthors(fallbackAuthors)),
lookupAuthors: dedupeAuthors(normalizeAuthors(lookupAuthors)),
githubLogins: contributors.map((contributor) => cleanString(contributor?.login ?? '').toLowerCase()).filter(Boolean),
};
}
105 changes: 105 additions & 0 deletions tests/services/commitAuthors.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import test from 'node:test';
import assert from 'node:assert/strict';

import { fetchCommitAuthors } from '../../src/services/githubImporterCommitAuthors.js';
import { cleanString, normalizeAuthor, normalizeAuthors } from '../../src/services/githubImporterUtils.js';

function commit(name, login = '') {
return { commit: { author: { name } }, author: login ? { login, type: 'User' } : null };
}

test('authenticated scans continue to page two for human author names', async () => {
const urls = [];
const warnings = [];
const result = await fetchCommitAuthors({
owner: 'test-owner', repo: 'test-repo', defaultBranch: 'feature/new', authToken: 'token',
initialCommits: Array.from({ length: 100 }, () => commit('known-login')),
knownGithubLogins: ['known-login'], warnings,
cleanString, normalizeAuthor, normalizeAuthors,
addWarning: (items, source, code) => items.push({ source, code }),
fetchOptionalJson: async (url) => {
urls.push(url);
return [commit('Ada Lovelace'), commit('bot-123'), commit('name-handle'), commit('Alice Example', 'Alice Example')];
},
});
assert.equal(urls.length, 1);
assert.equal(urls[0].endsWith('commits?per_page=100&sha=feature%2Fnew&page=2'), true);
assert.deepEqual(result.map((author) => `${author.givenNames} ${author.familyNames}`), ['Ada Lovelace']);
assert.equal(warnings.length, 0);
});

test('commit names matching logins on any scanned page are excluded', async () => {
const initialCommits = [
commit('Jane-Doe'),
commit('Anna-Maria'),
commit('Mary Smith-Jones'),
...Array.from({ length: 97 }, () => commit('Repeated Person')),
];
const result = await fetchCommitAuthors({
owner: 'test-owner',
repo: 'test-repo',
defaultBranch: 'main',
authToken: 'token',
initialCommits,
knownGithubLogins: [],
warnings: [],
cleanString,
normalizeAuthor,
normalizeAuthors,
addWarning: () => {},
fetchOptionalJson: async () => [
commit('Real Author', 'jane-doe'),
commit('Another Contributor', 'ANNA-MARIA'),
commit('Jean-Luc'),
commit('Full Name', 'jean-luc'),
],
});

assert.deepEqual(result.map((author) => `${author.givenNames} ${author.familyNames}`), [
'Mary Smith Jones',
'Repeated Person',
'Real Author',
'Another Contributor',
'Full Name',
]);
});

test('unauthenticated scans stop after the first 100 commits', async () => {
const warnings = [];
const result = await fetchCommitAuthors({
owner: 'test-owner', repo: 'test-repo', initialCommits: Array.from({ length: 100 }, () => commit('Dana Example')),
warnings, cleanString, normalizeAuthor, normalizeAuthors,
addWarning: (items, source, code) => items.push({ source, code }),
fetchOptionalJson: async () => { throw new Error('Should not request another page'); },
});
assert.equal(result.length, 1);
assert.equal(warnings.some((warning) => warning.code === 'commit-author-scan-limited'), true);
});

test('commit history excludes automated identities and aliases but accepts full hyphenated names', async () => {
const result = await fetchCommitAuthors({
owner: 'test-owner',
repo: 'test-repo',
initialCommits: [
commit('Jean-Pierre Dupont'),
commit('Mary Smith-Jones'),
commit('Claude Code'),
commit('GitHub Actions'),
commit('GitHub Actions', 'actions-bot'),
commit('Claude Fable'),
commit('alice_dev'),
commit('dev@example.com'),
],
warnings: [],
cleanString,
normalizeAuthor,
normalizeAuthors,
addWarning: () => {},
fetchOptionalJson: async () => [],
});

assert.deepEqual(result.map((author) => `${author.givenNames} ${author.familyNames}`), [
'Jean Pierre Dupont',
'Mary Smith Jones',
]);
});
Loading
Loading