Skip to content

fix(test): isolate Unicode generator checks from moving perl5 #1192

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fglock merged 1 commit into from
Aug 29, 2026
Merged

fix(test): isolate Unicode generator checks from moving perl5 #1192

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 0 additions & 43 deletions src/test/java/org/perlonjava/runtime/regex/PerlUnicodeBlockDataTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,10 @@
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assumptions.assumeTrue;

import com.ibm.icu.text.UnicodeSet;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -125,44 +120,6 @@ void followsPerlLoosePropertyAliasRules() {
assertFalse(PerlUnicodeBlockData.isPropertyAlias(null));
}

@Test
void generatorReproducesCheckedInDataByteForByte() throws Exception {
Path generator = Path.of("dev", "regex", "tools",
"generate_perl_unicode_block_data.pl");
Path generated = Path.of("src", "main", "java", "org", "perlonjava", "runtime",
"regex", "PerlUnicodeBlockData.java");
Process process = new ProcessBuilder("perl", generator.toString())
.redirectErrorStream(true)
.start();
byte[] actual = process.getInputStream().readAllBytes();
assertTrue(process.waitFor(30, TimeUnit.SECONDS), "generator timed out");
if (process.exitValue() != 0
&& !hasExplicitGeneratorSource()
&& reportsMissingDevelopmentSources(actual)) {
assumeTrue(false, new String(actual, StandardCharsets.UTF_8).trim());
}
assertEquals(0, process.exitValue(), new String(actual, StandardCharsets.UTF_8));
assertArrayEquals(Files.readAllBytes(generated), actual);
for (byte value : actual) {
assertFalse(value == '\r', "generated source must use raw LF");
}
}

private static boolean hasExplicitGeneratorSource() {
return hasText(System.getenv("PERLONJAVA_UNICODE_ROOT"))
|| hasText(System.getenv("PERLONJAVA_PERL_ROOT"));
}

private static boolean hasText(String value) {
return value != null && !value.isBlank();
}

private static boolean reportsMissingDevelopmentSources(byte[] output) {
String diagnostic = new String(output, StandardCharsets.UTF_8);
return diagnostic.startsWith("No complete Unicode current source tree:")
|| diagnostic.startsWith("No complete current Perl source tree:");
}

private static UnicodeSet set(String alias) {
UnicodeSet set = PerlUnicodeBlockData.set(alias);
if (set == null) throw new AssertionError("Unknown Block alias " + alias);
Expand Down
27 changes: 0 additions & 27 deletions src/test/java/org/perlonjava/runtime/regex/PerlUnicodeDecompositionTypeDataTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,6 @@
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -167,26 +162,4 @@ public void acceptsAllLooseAliasesIncludingPerlUnionValues() {
PerlUnicodeDecompositionTypeData.NONE, PerlUnicodeDecompositionTypeData.NON_CANONICAL));
}

@Test
public void generatorReproducesTheCheckedInClassByteForByte() throws Exception {
Path root = Path.of(System.getProperty("user.dir"));
Path generator = root.resolve("dev/regex/tools/generate_perl_unicode_decomposition_type_data.pl");
Path generated = root.resolve("src/main/java/org/perlonjava/runtime/regex/PerlUnicodeDecompositionTypeData.java");

Process process = new ProcessBuilder("perl", generator.toString())
.directory(root.toFile())
.redirectErrorStream(true)
.start();
ByteArrayOutputStream output = new ByteArrayOutputStream();
process.getInputStream().transferTo(output);
assertEquals(0, process.waitFor(), new String(output.toByteArray()));
assertArrayEquals(canonicalLf(Files.readAllBytes(generated)),
canonicalLf(output.toByteArray()));
}

private static byte[] canonicalLf(byte[] bytes) {
return new String(bytes, StandardCharsets.UTF_8)
.replace("\r\n", "\n")
.getBytes(StandardCharsets.UTF_8);
}
}
29 changes: 0 additions & 29 deletions src/test/java/org/perlonjava/runtime/regex/PerlUnicodeNamedSequenceDataTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,6 @@
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;

import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

Expand All @@ -29,28 +24,4 @@ void usesPinnedPerlUnicode17NamedSequences() {
assertNull(PerlUnicodeNamedSequenceData.sequence(null));
}

@Test
void generatorReproducesTheCheckedInClassByteForByte() throws Exception {
Path root = Path.of(System.getProperty("user.dir"));
Path generator = root.resolve(
"dev/regex/tools/generate_perl_unicode_named_sequence_data.pl");
Path generated = root.resolve(
"src/main/java/org/perlonjava/runtime/regex/PerlUnicodeNamedSequenceData.java");

Process process = new ProcessBuilder("perl", generator.toString())
.directory(root.toFile())
.redirectErrorStream(true)
.start();
ByteArrayOutputStream output = new ByteArrayOutputStream();
process.getInputStream().transferTo(output);
assertEquals(0, process.waitFor(), new String(output.toByteArray()));
assertArrayEquals(canonicalLf(Files.readAllBytes(generated)),
canonicalLf(output.toByteArray()));
}

private static byte[] canonicalLf(byte[] bytes) {
return new String(bytes, StandardCharsets.UTF_8)
.replace("\r\n", "\n")
.getBytes(StandardCharsets.UTF_8);
}
}
21 changes: 1 addition & 20 deletions src/test/java/org/perlonjava/runtime/regex/PerlUnicodeNumericValueDataTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,7 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -120,21 +115,6 @@ void followsPerlLoosePropertyAliasRules() {
assertFalse(PerlUnicodeNumericValueData.isPropertyAlias(null));
}

@Test
void generatorReproducesCheckedInDataByteForByte() throws Exception {
Path generator = Path.of("dev", "regex", "tools",
"generate_perl_unicode_numeric_value_data.pl");
Path generated = Path.of("src", "main", "java", "org", "perlonjava", "runtime",
"regex", "PerlUnicodeNumericValueData.java");
Process process = new ProcessBuilder("perl", generator.toString())
.redirectErrorStream(true)
.start();
byte[] actual = process.getInputStream().readAllBytes();
assertTrue(process.waitFor(30, TimeUnit.SECONDS), "generator timed out");
assertEquals(0, process.exitValue(), new String(actual, StandardCharsets.UTF_8));
assertArrayEquals(Files.readAllBytes(generated), actual);
}

private static com.ibm.icu.text.UnicodeSet set(String canonicalValue) {
return PerlUnicodeNumericValueData.set(index(canonicalValue));
}
Expand All @@ -147,4 +127,5 @@ private static short index(String canonicalValue) {
}
throw new AssertionError("Unknown Numeric_Value " + canonicalValue);
}

}
34 changes: 0 additions & 34 deletions third_party/joni/test/org/joni/TestPerlUnicodeCaseFoldData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,9 @@
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.io.ByteArrayOutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Set;

import org.junit.Assume;
import org.junit.Test;

public class TestPerlUnicodeCaseFoldData {
Expand Down Expand Up @@ -219,36 +215,6 @@ public void rejectsUnknownLookupsAndInvalidSlices() {
() -> PerlUnicodeCaseFoldData.reverseSequenceLengthAt(73));
}

@Test
public void generatorReproducesCheckedInDataByteForByteWhenSourcesExist() throws Exception {
Path root = Path.of(System.getProperty("user.dir"));
Path perlRoot = selectPerlRoot(root);
Path unicodeRoot = perlRoot.resolve("lib/unicore");
Assume.assumeTrue(Files.isRegularFile(unicodeRoot.resolve("CaseFolding.txt")));
Assume.assumeTrue(Files.isRegularFile(perlRoot.resolve(
"regen/regcharclass_multi_char_folds.pl")));

Path generator = root.resolve("dev/regex/tools/generate_perl_unicode_case_fold_data.pl");
ProcessBuilder builder = new ProcessBuilder("perl", generator.toString())
.directory(root.toFile())
.redirectErrorStream(true);
builder.environment().put("PERLONJAVA_PERL_ROOT", perlRoot.toString());
builder.environment().put("PERLONJAVA_UNICODE_ROOT", unicodeRoot.toString());
Process process = builder.start();
ByteArrayOutputStream output = new ByteArrayOutputStream();
process.getInputStream().transferTo(output);
assertEquals(new String(output.toByteArray()), 0, process.waitFor());
assertArrayEquals(Files.readAllBytes(root.resolve(
"third_party/joni/src/org/joni/PerlUnicodeCaseFoldData.java")),
output.toByteArray());
}

private static Path selectPerlRoot(Path root) {
String configured = System.getenv("PERLONJAVA_PERL_ROOT");
return configured == null || configured.isEmpty()
? root.resolve("perl5") : Path.of(configured);
}

private static void assertFullFold(int source, int... expected) {
assertEquals(expected.length, PerlUnicodeCaseFoldData.fullFoldLength(source));
int[] actual = new int[expected.length];
Expand Down
Loading