From 30079d5f5f002aad16a426f9561b558c9b326d88 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Fri, 16 Oct 2026 20:04:08 +0000 Subject: [PATCH 1/8] Add pluggable WAL storage providers Introduce the WalProvider SPI and properties to configure the available set of WAL providers and the current provider. This commit converts the current DFS implementation to a DfsWalProvider and the upgrade code converts all existing wal entries in ZooKeeper and the root and metadata tables to use the 'dfs' provider id. Post-upgrade and on instance restart the `instance.wal.provider' property can be set to the class name of a WALProvider SPI implementation. The 'instance.wal.providers' property must contain the class name and configuration properties for the current and all in-use providers. An InMemoryWalProvider class in the test module provides an example implementation and is used in various ITs. Assisted-By: OpenAI GPT-6 Luna --- WAL-SPI.md | 188 ++++++++++++ assemble/conf/accumulo.properties | 6 + .../apache/accumulo/core/conf/Property.java | 12 + .../accumulo/core/spi/wal/WalProvider.java | 191 ++++++++++++ .../core/tabletserver/log/LogEntry.java | 31 +- .../accumulo/core/conf/PropertyTest.java | 16 + .../metadata/schema/TabletMetadataTest.java | 18 +- .../core/tabletserver/log/LogEntryTest.java | 48 ++- .../MiniAccumuloClusterImpl.java | 11 + server/base/pom.xml | 4 + .../apache/accumulo/server/ServerContext.java | 7 + .../apache/accumulo/server/fs/VolumeUtil.java | 10 +- .../accumulo/server/log/DfsWalProvider.java | 182 ++++++++++++ .../server/log/WalProviderRegistry.java | 178 +++++++++++ .../accumulo/server/log/WalStateManager.java | 64 +++- .../state/AbstractTabletStateStore.java | 14 +- .../state/LoggingTabletStateStore.java | 6 +- .../manager/state/TabletStateStore.java | 10 +- .../manager/state/ZooTabletStateStore.java | 6 +- .../checkCommand/SystemConfigCheckRunner.java | 2 +- .../accumulo/server/fs/VolumeUtilTest.java | 7 +- .../server/log/WalProviderRegistryTest.java | 62 ++++ .../server/log/WalStateManagerTest.java | 91 ++++++ .../manager/state/TabletManagementTest.java | 7 +- .../src/test/resources/accumulo.properties | 2 + .../gc/GarbageCollectWriteAheadLogs.java | 20 +- .../accumulo/gc/SimpleGarbageCollector.java | 1 + .../gc/GarbageCollectWriteAheadLogsTest.java | 12 +- .../gc/src/test/resources/accumulo.properties | 2 + .../accumulo/manager/BalanceManager.java | 2 +- .../org/apache/accumulo/manager/Manager.java | 42 +++ .../accumulo/manager/TabletGroupWatcher.java | 12 +- .../manager/recovery/RecoveryManager.java | 31 +- .../manager/upgrade/Upgrader11to12.java | 102 ++++++- .../tableOps/merge/MergeTabletsTest.java | 4 +- .../tableOps/split/UpdateTabletsTest.java | 4 +- .../manager/upgrade/Upgrader11to12Test.java | 77 +++++ .../src/test/resources/accumulo.properties | 2 + .../apache/accumulo/tserver/ScanServer.java | 1 + .../apache/accumulo/tserver/TabletServer.java | 9 +- .../accumulo/tserver/log/DfsLogger.java | 151 ++++------ .../accumulo/tserver/log/LogSorter.java | 39 +-- .../tserver/log/TabletServerLogger.java | 3 +- .../accumulo/tserver/logger/LogReader.java | 4 +- .../accumulo/tserver/WalRemovalOrderTest.java | 4 +- .../tserver/log/RecoveryLogsIteratorTest.java | 4 +- .../tserver/log/SortedLogRecoveryTest.java | 4 +- .../src/test/resources/accumulo.properties | 2 + shell/pom.xml | 5 + .../commands/ListTabletsCommandTest.java | 7 +- .../MissingWalHeaderCompletesRecoveryIT.java | 7 +- .../test/fate/ManagerRepoIT_SimpleSuite.java | 4 +- .../functional/AmpleConditionalWriterIT.java | 13 +- .../functional/InMemoryWalProviderIT.java | 144 +++++++++ .../TabletManagementIteratorIT.java | 3 +- .../test/functional/WalProviderSwitchIT.java | 172 +++++++++++ .../test/wal/InMemoryWalProvider.java | 277 ++++++++++++++++++ .../accumulo/test/wal/InMemoryWalStore.java | 223 ++++++++++++++ test/src/main/resources/accumulo.properties | 2 + .../test/wal/InMemoryWalProviderTest.java | 118 ++++++++ 60 files changed, 2456 insertions(+), 224 deletions(-) create mode 100644 WAL-SPI.md create mode 100644 core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java create mode 100644 server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java create mode 100644 server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java create mode 100644 server/base/src/test/java/org/apache/accumulo/server/log/WalProviderRegistryTest.java create mode 100644 server/base/src/test/java/org/apache/accumulo/server/log/WalStateManagerTest.java create mode 100644 test/src/main/java/org/apache/accumulo/test/functional/InMemoryWalProviderIT.java create mode 100644 test/src/main/java/org/apache/accumulo/test/functional/WalProviderSwitchIT.java create mode 100644 test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java create mode 100644 test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalStore.java create mode 100644 test/src/test/java/org/apache/accumulo/test/wal/InMemoryWalProviderTest.java diff --git a/WAL-SPI.md b/WAL-SPI.md new file mode 100644 index 00000000000..1f872a67f69 --- /dev/null +++ b/WAL-SPI.md @@ -0,0 +1,188 @@ + +# Write-ahead log storage SPI (current implementation) + +Accumulo's WAL storage SPI lets a TabletServer create raw write-ahead logs using a configured +`WalProvider`. The provider owns storage of the raw byte stream; Accumulo owns the WAL format, +encryption, tablet bookkeeping, sorting, and replay. The implementation is split between the +[SPI interface][spi], the [provider registry][registry], and the built-in +[filesystem provider][dfs-provider]. + +## Configuration and provider identity + +Two instance-wide properties configure providers in `accumulo.properties`: + +| Property | Purpose | +| ------------------------ | ------------------------------------------------------------- | +| `instance.wal.providers` | JSON map of provider classes to their option maps. | +| `instance.wal.provider` | Class for **new** WALs; must be a key in the map above. | + +Both properties are required. `instance.wal.providers` defaults to an empty map and +`instance.wal.provider` defaults to an empty string. For the built-in provider, configure +`org.apache.accumulo.server.log.DfsWalProvider` with an empty option map: + +```properties +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider +``` + +For a provider with options, put string-valued options in its map, for example +`{"com.example.MyWalProvider":{"endpoint":"https://wal.example"}}`. The +[distribution configuration][dist-config] shows these settings; their definitions are in +[Property.java][property]. + +Each implementation needs a public no-argument constructor and a stable `getId()` matching +`[A-Za-z0-9]+`. The ID `dfs` is reserved for `DfsWalProvider` and legacy filesystem WALs. The +[registry][registry] loads each configured class, passes its options to `init(InitParameters)` once per +server process, rejects duplicate provider IDs, and selects the current class by class name. The +default `init(InitParameters)` implementation calls `init(ServiceEnvironment)`, preserving providers +that implement the older initialization method. A +[memoized registry][context] is accessed by the Manager, TabletServers, ScanServers, and GC. The +[Manager][manager] also checks WAL IDs in tablet metadata and ZooKeeper markers against the +configured IDs at startup. + +Changing the current provider affects newly created logs only. Keep old providers in +`instance.wal.providers` while their WALs are still referenced, so recovery and cleanup can use +them. Provider IDs and configuration must remain consistent across server processes. + +## Upgrade of existing WAL references + +The [11-to-12 upgrader][upgrader] assigns the built-in provider ID `dfs` to pre-SPI filesystem +WALs without moving or rewriting the WAL files: + +- Legacy tablet log qualifiers `-/` become `dfs/`: the upgrader adds the new + qualifier and deletes the old one. This covers the root tablet metadata stored in ZooKeeper, + metadata-tablet references, and user-tablet references. Already converted qualifiers are left + alone. +- Legacy ZooKeeper WAL marker data `,` becomes `,dfs,`. + Markers already containing a provider ID are retained. The upgrader checks marker format and + verifies that an updated marker was persisted. +- The upgrader checks for remaining legacy `-/` log qualifiers in the metadata tables; it also + verifies the root tablet's WAL references. Malformed markers or leftover legacy qualifiers cause + the upgrade to fail rather than silently treating them as another provider's logs. + +Configure `DfsWalProvider` in `instance.wal.providers` during the upgrade and retain it while old +WAL references or markers exist, even if a different class is selected in `instance.wal.provider` +for new writes. Otherwise the Manager's provider-reference check or subsequent recovery and GC +cannot handle those existing `dfs` references. The upgrade changes **reference metadata**, not the +raw WAL format or the sorted-log recovery process. + +## Provider contract + +[`WalProvider`][spi] exposes: + +| Method | Use | +| ------------------------------- | ----------------------------------------------------- | +| `create(serverAddress, UUID)` | Exclusively create a WAL and return its `Writer`. | +| `open(reference)` | Read a WAL from byte zero for sorting/recovery. | +| `exists(reference)` | Check a WAL, including while it may still be written. | +| `delete(reference)` | Remove an unreferenced WAL; should be idempotent. | +| `getId()` / `init(environment)` | Supply the persisted ID and initialize the provider. | + +The nested `Writer` supplies `getReference()`, `getOutputStream()`, `sync(Durability)`, +`getLength()`, and `close()`. It can optionally report `getCurrentReplication()` and +`getPipeline()` for diagnostics (the defaults return `-1` and an empty array). Accumulo writes the +WAL header and records to the output stream: a provider must retain the exact bytes in order and +return them through `open()`. Providers must be thread-safe for operations on different logs; a +writer must also coordinate concurrent appends and sync requests. `FLUSH` must survive a +TabletServer process failure, `SYNC` must provide the synchronization guarantee expected of synced +WAL writes, and `LOG` does not invoke `Writer.sync()`. Closing a writer must be idempotent. + +The reference returned by `Writer.getReference()` is persisted and must stay usable by that +provider's `open()` and `delete()`. It must be a fully qualified Hadoop path in the standard WAL +layout, with a final `/` suffix (the address's `:` becomes `+`). The current +[recovery path conversion][rpath] also expects a `wal` directory immediately before the +server component; it maps `/wal//` to `/recovery/`. A custom +provider can use a custom Hadoop filesystem URI, but this path structure and access to the +corresponding recovery destination are current integration constraints. + +## Writing and tracking a WAL + +1. [`TabletServerLogger`][tslog] obtains `getCurrent()` from the registry when creating a + log. Despite its name, [`DfsLogger`][dfs-logger] uses the selected provider's `create()` and + `Writer`, so the WAL need not be stored by `DfsWalProvider`. +2. `DfsLogger` validates the returned reference as a [`LogEntry`][log-entry], writes the versioned + header and crypto parameters, wraps the provider output stream for encryption, writes an `OPEN` + entry, and syncs it before using the log. Subsequent tablet definitions, mutations, and + compaction events use the existing WAL serialization. A background sync task batches durability + requests and calls `Writer.sync(FLUSH)` or `Writer.sync(SYNC)`; it uses the optional + replication/pipeline methods for diagnostics. +3. `LogEntry` associates the provider ID with the path. Tablet metadata stores it as a log-column + qualifier of the form `/`. [`WalStateManager`][wal-state] records the WAL + state (`OPEN`, `CLOSED`, or `UNREFERENCED`), provider ID, and path in ZooKeeper WAL markers. This + lets recovery and GC identify which provider created a given log, independently of the provider + currently selected for new writes. + +[`DfsWalProvider`][dfs-provider] is the built-in implementation. It chooses an Accumulo volume and +creates a filesystem-backed WAL under `wal//` using `VolumeManager`; its writer +uses `hflush()` for `FLUSH` and `hsync()` for `SYNC`. It opens and checks files through +`VolumeManager`, and uses the filesystem trash or recursive deletion for cleanup. Where the +underlying stream is an HDFS `DFSOutputStream`, it can report replication and pipeline information. + +## Recovery and garbage collection + +The [Manager's `RecoveryManager`][rm] obtains the provider ID from the `LogEntry`, +derives the sibling `recovery/` directory using `RecoveryPath`, and queues a sort. Filesystem +WALs with ID `dfs` use the configured Hadoop log closer before sorting; other providers do not use +that closer. The queued work carries the provider ID, source reference, and recovery destination. + +[`LogSorter`][sorter] opens the raw source through `VolumeManager` for `dfs` or through the +registered provider's `open(reference)` for another ID. It decrypts and decodes Accumulo's WAL +records, sorts them into RFiles in the recovery directory via `VolumeManager`, and writes a finished +marker. The sorted recovery files are **not** read through `WalProvider.open()`: +[`ResolvedSortedLog`][resolved-log] validates that directory and marker, +[`RecoveryLogsIterator`][rli] opens the RFiles using the filesystem selected by +`VolumeManager`, and [`SortedLogRecovery`][slr] replays the tablet's applicable +mutations. + +Thus the SPI permits alternative storage for the **raw WAL**, but it does not replace the existing +filesystem-based sorted-log recovery pipeline. HDFS itself is not a requirement of +`SortedLogRecovery`: its [unit test][slr-test] uses a local `file://` filesystem. The recovery +destination must nevertheless be accessible through `VolumeManager`, and the reference must satisfy +the path layout above. + +After WALs are no longer referenced, [`GarbageCollectWriteAheadLogs`][wal-gc] deletes raw `dfs` +logs through `VolumeManager` and other raw logs through their registered provider's +`delete(reference)`. It removes sorted recovery directories through `VolumeManager` separately. +The [system configuration check][c] uses the provider's `exists(reference)` when +checking WAL markers. + +[spi]: core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java +[registry]: server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java +[dfs-provider]: server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java +[dist-config]: assemble/conf/accumulo.properties +[property]: core/src/main/java/org/apache/accumulo/core/conf/Property.java +[context]: server/base/src/main/java/org/apache/accumulo/server/ServerContext.java +[manager]: server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +[upgrader]: server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java +[rpath]: server/base/src/main/java/org/apache/accumulo/server/manager/recovery/RecoveryPath.java +[tslog]: server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java +[dfs-logger]: server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java +[log-entry]: core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java +[wal-state]: server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java +[rm]: server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java +[sorter]: server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java +[resolved-log]: server/tserver/src/main/java/org/apache/accumulo/tserver/log/ResolvedSortedLog.java +[rli]: server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogsIterator.java +[slr]: server/tserver/src/main/java/org/apache/accumulo/tserver/log/SortedLogRecovery.java +[slr-test]: server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java +[wal-gc]: server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java +[c]: server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java diff --git a/assemble/conf/accumulo.properties b/assemble/conf/accumulo.properties index 6e6de34cc63..5a9532f1d02 100644 --- a/assemble/conf/accumulo.properties +++ b/assemble/conf/accumulo.properties @@ -23,6 +23,12 @@ ## Sets location in HDFS where Accumulo will store data instance.volumes=hdfs://localhost:8020/accumulo +## Configure the WAL provider implementation consistently on every server +## For the built-in filesystem-backed implementation, configure: +## org.apache.accumulo.server.log.DfsWalProvider +# instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +# instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider + ## Sets location of Zookeepers instance.zookeeper.host=localhost:2181 diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index cec1a3f9649..aafef25bef4 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -260,6 +260,18 @@ temporary files (for example, when creating a pre-split table). \ + "encryption, replace this classname with an implementation of the" + "org.apache.accumulo.core.spi.crypto.CryptoFactory interface.", "2.1.0"), + INSTANCE_WAL_PROVIDERS("instance.wal.providers", "{}", PropertyType.JSON, + "Required JSON object mapping fully qualified class names implementing " + + "org.apache.accumulo.core.spi.wal.WalProvider to string-valued option maps. Use an " + + "empty map when a provider has no options. All configured providers are validated at startup, must be " + + "available to Managers, TabletServers, ScanServers, and GC, and must include providers " + + "referenced by existing WALs. This must be consistent across the instance.", + "4.0.0"), + INSTANCE_WAL_PROVIDER("instance.wal.provider", "", PropertyType.CLASSNAME, + "Required. The class implementing org.apache.accumulo.core.spi.wal.WalProvider used to create new " + + "WALs. It must also be listed in instance.wal.providers and must be consistent " + + "across the instance.", + "4.0.0"), // general properties GENERAL_PREFIX("general.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of accumulo overall, but" diff --git a/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java new file mode 100644 index 00000000000..842c3dca1b2 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.spi.wal; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Map; +import java.util.UUID; + +import org.apache.accumulo.core.client.Durability; +import org.apache.accumulo.core.spi.common.ServiceEnvironment; + +/** + * Storage provider for TabletServer write-ahead logs. Implementations are initialized once per + * server process and must be thread-safe because Accumulo may create, read, and delete different + * logs concurrently. The provider stores an opaque byte stream: Accumulo writes the log header and + * records to the returned output stream, and a reader must return those exact bytes in their + * original write order. Configured implementations must have a public no-argument constructor. + * + *

+ * A provider ID is persisted with every log reference. It must match {@code [A-Za-z0-9]+} and + * remain stable for as long as any log created by the provider can be referenced. Every provider + * that may be needed by existing tablet metadata or WAL markers must be configured as available. + * + *

+ * To keep Accumulo's existing recovery pipeline, log references must be fully qualified Hadoop + * paths in the standard WAL layout, ending in {@code /}. Providers may use a + * custom Hadoop filesystem URI, but must preserve this path layout. + * + * @since 4.0.0 + */ +public interface WalProvider { + + /** Regular expression for valid provider IDs. */ + String PROVIDER_ID_REGEX = "[A-Za-z0-9]+"; + + /** + * Returns the stable identifier persisted with logs created by this provider. It must be unique + * among the configured providers and match {@link #PROVIDER_ID_REGEX}. + * + * @return stable provider ID + */ + String getId(); + + /** + * Initializes the provider once after construction. Implementations may retain the environment + * for accessing system configuration and must not assume that properties change during this + * process lifetime. Throwing an exception prevents the server process from starting. + * + * @param environment system configuration and Accumulo service information + */ + default void init(ServiceEnvironment environment) {} + + /** + * Initializes this provider with its configured options. The default implementation calls + * {@link #init(ServiceEnvironment)} for compatibility with providers that only implement that + * method. + * + * @param params the service environment and options configured for this provider + * @since 4.0.0 + */ + default void init(InitParameters params) { + init(params.getServiceEnvironment()); + } + + /** Parameters passed to {@link #init(InitParameters)}. */ + interface InitParameters { + + /** @return system configuration and Accumulo service information */ + ServiceEnvironment getServiceEnvironment(); + + /** @return the provider-specific options configured in {@code instance.wal.providers} */ + Map getOptions(); + } + + /** + * Creates a new exclusive WAL for the given TabletServer and UUID. The returned reference must be + * unique, fully qualified, and end in the supplied server address (with {@code :} replaced by + * {@code +}) followed by the supplied UUID. The writer must preserve every byte written to its + * output stream in order. + * + * @param serverAddress advertised TabletServer address in {@code host:port} form + * @param id globally unique identifier for the WAL + * @return a new writer whose reference can later be passed to {@link #open(String)} and + * {@link #delete(String)} + */ + Writer create(String serverAddress, UUID id) throws IOException; + + /** + * Opens a completed or abandoned WAL for recovery. The returned stream must start at byte zero + * and reproduce the bytes written by the corresponding writer exactly and in the same order. + * Accumulo only opens logs after their TabletServer has closed or lost ownership of them. + * + * @param reference reference returned by {@link Writer#getReference()} + */ + InputStream open(String reference) throws IOException; + + /** + * Checks whether a WAL reference currently exists without opening it. This may be called while a + * WAL is still being written. + * + * @param reference reference returned by {@link Writer#getReference()} + * @return true if the WAL exists + */ + boolean exists(String reference) throws IOException; + + /** + * Deletes a WAL that Accumulo has determined is no longer referenced by any tablet. Deletion + * should be idempotent because a server may retry cleanup after an interruption. + * + * @param reference reference returned by {@link Writer#getReference()} + */ + void delete(String reference) throws IOException; + + /** + * A newly created WAL and its append-only output stream. Accumulo may call + * {@link #sync(Durability)} while other threads append through the output stream; implementations + * must ensure a sync covers all bytes written before it is requested. + */ + interface Writer extends AutoCloseable { + + /** + * Returns the stable, fully qualified reference for this WAL. It is persisted in tablet + * metadata and WAL markers and must remain usable by {@link WalProvider#open(String)} and + * {@link WalProvider#delete(String)}. + */ + String getReference(); + + /** + * Returns the stream Accumulo writes the WAL header and records to. Accumulo owns and closes + * this stream through {@link #close()}; implementations may buffer internally but must preserve + * every byte and its order, and make buffered bytes readable after the corresponding sync or + * close completes. + */ + OutputStream getOutputStream(); + + /** + * Makes all preceding bytes durable according to Accumulo's requested durability. {@code FLUSH} + * must survive a TabletServer process failure and {@code SYNC} must satisfy the filesystem + * synchronization guarantee used by Accumulo for synced WAL writes. This method is not called + * for {@code LOG} durability. + */ + void sync(Durability durability) throws IOException; + + /** + * Returns the number of bytes written so far, including the WAL header, for TabletServer WAL + * metrics and diagnostics. This method may run concurrently with writes. + */ + long getLength() throws IOException; + + /** + * Returns the current replication level when the provider can report one, or {@code -1} when + * that information is unavailable. + */ + default int getCurrentReplication() throws IOException { + return -1; + } + + /** + * Returns a diagnostic description of the current storage pipeline, or an empty array when the + * provider has no pipeline information. + */ + default String[] getPipeline() { + return new String[0]; + } + + /** + * Closes the WAL and releases all associated resources. Closing must be idempotent because + * Accumulo may close a writer after an initialization error. + */ + @Override + void close() throws IOException; + } +} diff --git a/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java b/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java index 57820065b07..b9706f6afb5 100644 --- a/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java +++ b/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java @@ -26,6 +26,7 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; +import org.apache.accumulo.core.spi.wal.WalProvider; import org.apache.hadoop.io.Text; import org.checkerframework.checker.nullness.qual.NonNull; @@ -36,12 +37,15 @@ public final class LogEntry { private final String path; + private final String providerId; private final HostAndPort tserver; private final UUID uniqueId; private final Text columnQualifier; - private LogEntry(String path, HostAndPort tserver, UUID uniqueId, Text columnQualifier) { + private LogEntry(String path, String providerId, HostAndPort tserver, UUID uniqueId, + Text columnQualifier) { this.path = path; + this.providerId = providerId; this.tserver = tserver; this.uniqueId = uniqueId; this.columnQualifier = columnQualifier; @@ -54,15 +58,19 @@ private LogEntry(String path, HostAndPort tserver, UUID uniqueId, Text columnQua * For example, file:///some/dir/path/localhost+1234/927ba659-d109-4bce-b0a5-bcbbcb9942a2 is a * valid path. * + * @param providerId provider that owns this WAL * @param path path to validate * @return an object representation of this log entry * @throws IllegalArgumentException if the path is invalid */ - public static LogEntry fromPath(String path) { - return validatedLogEntry(path, null); + public static LogEntry fromPath(String providerId, String path) { + return validatedLogEntry(path, providerId, null); } - private static LogEntry validatedLogEntry(String path, Text columnQualifier) { + private static LogEntry validatedLogEntry(String path, String providerId, Text columnQualifier) { + Preconditions.checkArgument( + providerId != null && providerId.matches(WalProvider.PROVIDER_ID_REGEX), + "Invalid WAL provider ID: %s", providerId); String[] parts = path.split("/"); if (parts.length < 2) { @@ -96,7 +104,7 @@ private static LogEntry validatedLogEntry(String path, Text columnQualifier) { throw new IllegalArgumentException(badUuidMsg); } - return new LogEntry(path, tserver, uuid, columnQualifier); + return new LogEntry(path, providerId, tserver, uuid, columnQualifier); } /** @@ -115,7 +123,7 @@ public static LogEntry fromMetaWalEntry(Entry entry) { Text qualifier = entry.getKey().getColumnQualifier(); String[] parts = qualifier.toString().split("/", 2); Preconditions.checkArgument(parts.length == 2, "Malformed write-ahead log %s", qualifier); - return validatedLogEntry(parts[1], qualifier); + return validatedLogEntry(parts[1], parts[0], qualifier); } @NonNull @@ -129,6 +137,10 @@ public String getPath() { return path; } + public String getProviderId() { + return providerId; + } + @NonNull public UUID getUniqueID() { return uniqueId; @@ -145,14 +157,15 @@ public boolean equals(Object other) { return true; } if (other instanceof LogEntry) { - return path.equals(((LogEntry) other).path); + return path.equals(((LogEntry) other).path) + && providerId.equals(((LogEntry) other).providerId); } return false; } @Override public int hashCode() { - return Objects.hash(path); + return Objects.hash(providerId, path); } /** @@ -163,7 +176,7 @@ public Text getColumnQualifier() { } private Text newCQ() { - return new Text("-/" + getPath()); + return new Text(providerId + "/" + getPath()); } /** diff --git a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java index be100685e85..df6add2f924 100644 --- a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java +++ b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java @@ -106,6 +106,22 @@ public void testProperties() { } } + @Test + public void testInstanceWalProviderProperties() { + assertEquals("instance.wal.providers", Property.INSTANCE_WAL_PROVIDERS.getKey()); + assertEquals("instance.wal.provider", Property.INSTANCE_WAL_PROVIDER.getKey()); + assertEquals("{}", Property.INSTANCE_WAL_PROVIDERS.getDefaultValue()); + assertEquals("", Property.INSTANCE_WAL_PROVIDER.getDefaultValue()); + assertEquals(PropertyType.JSON, Property.INSTANCE_WAL_PROVIDERS.getType()); + assertEquals(PropertyType.CLASSNAME, Property.INSTANCE_WAL_PROVIDER.getType()); + assertTrue(Property.isValidProperty(Property.INSTANCE_WAL_PROVIDERS.getKey(), + Property.INSTANCE_WAL_PROVIDERS.getDefaultValue())); + assertFalse(Property.isValidZooPropertyKey(Property.INSTANCE_WAL_PROVIDERS.getKey())); + assertFalse(Property.isFixedZooPropertyKey(Property.INSTANCE_WAL_PROVIDERS)); + assertFalse(Property.isValidZooPropertyKey(Property.INSTANCE_WAL_PROVIDER.getKey())); + assertFalse(Property.isFixedZooPropertyKey(Property.INSTANCE_WAL_PROVIDER)); + } + @Test public void testPorts() { HashSet usedPorts = new HashSet<>(); diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java index b56315d8d70..5aabea33b30 100644 --- a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java +++ b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java @@ -171,9 +171,9 @@ public void testAllColumns() { mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555"); - LogEntry le1 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + LogEntry le1 = LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()); le1.addToMutation(mutation); - LogEntry le2 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + LogEntry le2 = LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()); le2.addToMutation(mutation); StoredTabletFile sf1 = StoredTabletFile.of(new Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf")); @@ -515,8 +515,8 @@ public void testTmBuilderImmutable() { assertThrows(UnsupportedOperationException.class, () -> tm.getFilesMap().put(stf, new DataFileValue(0, 0, 0))); assertTrue(tm.getLogs().isEmpty()); - assertThrows(UnsupportedOperationException.class, - () -> tm.getLogs().add(LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()))); + assertThrows(UnsupportedOperationException.class, () -> tm.getLogs() + .add(LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()))); assertTrue(tm.getScans().isEmpty()); assertThrows(UnsupportedOperationException.class, () -> tm.getScans().add(stf)); assertTrue(tm.getLoaded().isEmpty()); @@ -534,7 +534,7 @@ public void testTmBuilderImmutable() { b.table(TableId.of("4")); b.extCompaction(ecid, ecMeta); b.file(stf, new DataFileValue(0, 0, 0)); - b.log(LogEntry.fromPath("localhost+8020/" + UUID.randomUUID())); + b.log(LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID())); b.scan(stf); b.loadedFile(stf, FateId.from(FateInstanceType.USER, UUID.randomUUID())); b.compacted(FateId.from(FateInstanceType.USER, UUID.randomUUID())); @@ -552,8 +552,8 @@ public void testTmBuilderImmutable() { assertThrows(UnsupportedOperationException.class, () -> tm2.getFilesMap().put(stf, new DataFileValue(0, 0, 0))); assertEquals(1, tm2.getLogs().size()); - assertThrows(UnsupportedOperationException.class, - () -> tm2.getLogs().add(LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()))); + assertThrows(UnsupportedOperationException.class, () -> tm2.getLogs() + .add(LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()))); assertEquals(1, tm2.getScans().size()); assertThrows(UnsupportedOperationException.class, () -> tm2.getScans().add(stf)); assertEquals(1, tm2.getLoaded().size()); @@ -829,8 +829,8 @@ public void testBuilder() { new CompactionMetadata(Set.of(sf1, sf2), rf1, "cid1", CompactionKind.USER, (short) 3, ResourceGroupId.of("Q1"), true, FateId.from(type, UUID.randomUUID())); - LogEntry le1 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); - LogEntry le2 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + LogEntry le1 = LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()); + LogEntry le2 = LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()); FateId selFilesFateId = FateId.from(type, UUID.randomUUID()); SelectedFiles selFiles = new SelectedFiles(Set.of(sf1, sf4), false, selFilesFateId, diff --git a/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java b/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java index 579571d81de..9da934cf385 100644 --- a/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java +++ b/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java @@ -29,6 +29,7 @@ import java.util.stream.Stream; import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; @@ -39,6 +40,7 @@ public class LogEntryTest { private final String validHost = "localhost+9997"; private final UUID validUUID = UUID.randomUUID(); + private static final String PROVIDER_ID = "example"; private final String validPath = "viewfs:/a/accumulo/wal/" + validHost + "/" + validUUID; @Test @@ -48,8 +50,8 @@ public void testColumnFamily() { @Test public void testFromPath() { - var logEntry = LogEntry.fromPath(validPath); - verifyLogEntry(logEntry, new Text("-/" + validPath)); + var logEntry = LogEntry.fromPath(PROVIDER_ID, validPath); + verifyLogEntry(logEntry, new Text(PROVIDER_ID + "/" + validPath)); } @Test @@ -60,6 +62,33 @@ public void testFromMetadata() { verifyLogEntry(logEntry, new Text(columnQualifier)); } + @Test + public void testProviderIdentityFromMetadata() { + var entry = LogEntry.fromPath("example", validPath); + var qualifier = entry.getColumnQualifier(); + var restored = LogEntry.fromMetaWalEntry(new SimpleImmutableEntry<>( + new Key("1<", LogColumnFamily.STR_NAME, qualifier.toString()), new Value())); + + assertEquals("example", restored.getProviderId()); + assertEquals(validPath, restored.getPath()); + assertEquals(entry, restored); + assertEquals(PROVIDER_ID, LogEntry.fromPath(PROVIDER_ID, validPath).getProviderId()); + } + + @Test + public void testLegacyMetadataQualifierRejected() { + var legacyEntry = new SimpleImmutableEntry<>( + new Key("1<", LogColumnFamily.STR_NAME, "-/" + validPath), new Value()); + assertThrows(IllegalArgumentException.class, () -> LogEntry.fromMetaWalEntry(legacyEntry)); + } + + @Test + public void testProviderIdMustBeAlphanumeric() { + assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath("provider-id", validPath)); + assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath("provider_id", validPath)); + assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath("provider.id", validPath)); + } + // helper for testing build from constructor or from metadata private void verifyLogEntry(LogEntry logEntry, Text expectedColumnQualifier) { assertEquals(validPath, logEntry.toString()); @@ -71,8 +100,8 @@ private void verifyLogEntry(LogEntry logEntry, Text expectedColumnQualifier) { @Test public void testEquals() { - LogEntry one = LogEntry.fromPath(validPath); - LogEntry two = LogEntry.fromPath(validPath); + LogEntry one = LogEntry.fromPath(PROVIDER_ID, validPath); + LogEntry two = LogEntry.fromPath(PROVIDER_ID, validPath); assertNotSame(one, two); assertEquals(one.toString(), two.toString()); @@ -93,13 +122,14 @@ public void testValidPaths() { var validPath3 = "dir2/" + validPath2; Stream.of(validPath1, validPath2, validPath3) - .forEach(s -> assertDoesNotThrow(() -> LogEntry.fromPath(s))); + .forEach(s -> assertDoesNotThrow(() -> LogEntry.fromPath(PROVIDER_ID, s))); } @Test public void testBadPathLength() { Stream.of("foo", "", validHost).forEach(badPath -> { - var e = assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath(badPath)); + var e = assertThrows(IllegalArgumentException.class, + () -> LogEntry.fromPath(PROVIDER_ID, badPath)); assertTrue(e.getMessage().contains("The path should end with tserver/UUID.")); }); } @@ -108,7 +138,8 @@ public void testBadPathLength() { public void testInvalidHostPort() { Stream.of("default:9997", "default+badPort").forEach(badHostAndPort -> { var badPath = badHostAndPort + "/" + validUUID; - var e = assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath(badPath)); + var e = assertThrows(IllegalArgumentException.class, + () -> LogEntry.fromPath(PROVIDER_ID, badPath)); assertTrue(e.getMessage().contains("Expected: host+port. Found '" + badHostAndPort + "'")); }); } @@ -117,7 +148,8 @@ public void testInvalidHostPort() { public void testInvalidUUID() { var badUUID = "badUUID"; var pathWithBadUUID = validHost + "/" + badUUID; - var e = assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath(pathWithBadUUID)); + var e = assertThrows(IllegalArgumentException.class, + () -> LogEntry.fromPath(PROVIDER_ID, pathWithBadUUID)); assertTrue(e.getMessage().contains("Expected valid UUID. Found '" + badUUID + "'")); } diff --git a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java index 9a4c4fd9bcd..2205125f283 100644 --- a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java +++ b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java @@ -101,6 +101,7 @@ import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.init.Initialize; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.util.AccumuloStatus; import org.apache.accumulo.server.util.PortUtils; import org.apache.accumulo.server.util.ZooZap; @@ -181,6 +182,16 @@ public MiniAccumuloClusterImpl(MiniAccumuloConfigImpl config) throws IOException config.setProperty(Property.MANAGER_TABLET_GROUP_WATCHER_INTERVAL, "5s"); } + if (!config.useExistingInstance()) { + if (!config.getSiteConfig().containsKey(Property.INSTANCE_WAL_PROVIDERS.getKey())) { + config.setProperty(Property.INSTANCE_WAL_PROVIDERS, + "{\"" + DfsWalProvider.class.getName() + "\":{}}"); + } + if (!config.getSiteConfig().containsKey(Property.INSTANCE_WAL_PROVIDER.getKey())) { + config.setProperty(Property.INSTANCE_WAL_PROVIDER, DfsWalProvider.class.getName()); + } + } + this.config = config.initialize(); this.clientProperties = Suppliers.memoize( () -> Accumulo.newClientProperties().from(config.getClientPropsFile().toPath()).build()); diff --git a/server/base/pom.xml b/server/base/pom.xml index 56a418487a3..098b5533a07 100644 --- a/server/base/pom.xml +++ b/server/base/pom.xml @@ -38,6 +38,10 @@ com.beust jcommander + + com.fasterxml.jackson.core + jackson-databind + com.github.ben-manes.caffeine caffeine diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java index 16b20f0d985..9949e94660e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerContext.java @@ -75,6 +75,7 @@ import org.apache.accumulo.server.conf.store.PropStore; import org.apache.accumulo.server.conf.store.impl.ZooPropStore; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.log.WalProviderRegistry; import org.apache.accumulo.server.mem.LowMemoryDetector; import org.apache.accumulo.server.metadata.ServerAmpleImpl; import org.apache.accumulo.server.metrics.MetricsInfoImpl; @@ -112,6 +113,7 @@ public class ServerContext extends ClientContext { private final Supplier sharedScheduledThreadPool; private final Supplier securityOperation; private final Supplier cryptoFactorySupplier; + private final Supplier walProviderRegistry; private final Supplier lowMemoryDetector; private final AtomicReference serverLock = new AtomicReference<>(); private final Supplier metricsInfoSupplier; @@ -148,6 +150,7 @@ private ServerContext(ServerInfo info, ResourceGroupId rgid) { secretManager = memoize(() -> new AuthenticationTokenSecretManager(getInstanceID(), getConfiguration().getTimeInMillis(Property.GENERAL_DELEGATION_TOKEN_LIFETIME))); cryptoFactorySupplier = memoize(() -> CryptoFactoryLoader.newInstance(getConfiguration())); + walProviderRegistry = memoize(() -> new WalProviderRegistry(this)); sharedScheduledThreadPool = memoize(() -> ThreadPools.getServerThreadPools() .createGeneralScheduledExecutorService(getConfiguration())); securityOperation = @@ -321,6 +324,10 @@ public CryptoServiceFactory getCryptoFactory() { return cryptoFactorySupplier.get(); } + public WalProviderRegistry getWalProviderRegistry() { + return walProviderRegistry.get(); + } + @Override public Ample getAmple() { return new ServerAmpleImpl(this); diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java index e41237b087b..6b961866ed2 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java @@ -34,6 +34,7 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.server.fs.VolumeManager.FileType; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; @@ -80,8 +81,12 @@ public static Path switchVolume(Path path, FileType ft, Map replaceme } public static LogEntry switchVolume(LogEntry le, Map replacements) { + if (!le.getProviderId().equals(DfsWalProvider.PROVIDER_ID)) { + return null; + } Path switchedPath = switchVolume(new Path(le.getPath()), FileType.WAL, replacements); - return switchedPath == null ? null : LogEntry.fromPath(switchedPath.toString()); + return switchedPath == null ? null + : LogEntry.fromPath(le.getProviderId(), switchedPath.toString()); } public static boolean needsVolumeReplacement(final Map replacements, @@ -130,6 +135,9 @@ public static void volumeReplacementEvaluation(final Map replacements log.trace("Using volume replacements: {}", replacements); for (LogEntry logEntry : tm.getLogs()) { + if (!logEntry.getProviderId().equals(DfsWalProvider.PROVIDER_ID)) { + continue; + } log.trace("Evaluating walog {} for replacement.", logEntry); LogEntry switchedLogEntry = switchVolume(logEntry, replacements); if (switchedLogEntry != null) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java new file mode 100644 index 00000000000..096c978b215 --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.log; + +import java.io.IOException; +import java.io.InputStream; +import java.util.UUID; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.Durability; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.spi.common.ServiceEnvironment; +import org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment; +import org.apache.accumulo.core.spi.wal.WalProvider; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.ServiceEnvironmentImpl; +import org.apache.accumulo.server.fs.VolumeChooserEnvironmentImpl; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSOutputStream; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; + +/** Filesystem-backed WAL provider. */ +public class DfsWalProvider implements WalProvider { + + /** Stable provider ID used for WALs stored on HDFS. */ + public static final String PROVIDER_ID = "dfs"; + + private ServerContext context; + + @Override + public String getId() { + return PROVIDER_ID; + } + + @Override + public void init(ServiceEnvironment environment) { + if (!(environment instanceof ServiceEnvironmentImpl serviceEnvironment)) { + throw new IllegalArgumentException("DfsWalProvider requires the server service environment"); + } + context = serviceEnvironment.getContext(); + } + + @Override + public Writer create(String serverAddress, UUID id) throws IOException { + String addressForFilename = serverAddress.replace(':', '+'); + var chooserEnv = + new VolumeChooserEnvironmentImpl(VolumeChooserEnvironment.Scope.LOGGER, context); + String logPath = + context.getVolumeManager().choose(chooserEnv, context.getBaseUris()) + Path.SEPARATOR + + Constants.WAL_DIR + Path.SEPARATOR + addressForFilename + Path.SEPARATOR + id; + var path = new Path(logPath); + VolumeManager fs = context.getVolumeManager(); + short replication = (short) context.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION); + if (replication == 0) { + replication = fs.getDefaultReplication(path); + } + long blockSize = context.getConfiguration().getAsBytes(Property.TSERV_WAL_BLOCKSIZE); + if (blockSize == 0) { + blockSize = (long) (context.getConfiguration().getAsBytes(Property.TSERV_WAL_MAX_SIZE) * 1.1); + } + FSDataOutputStream output; + if (context.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC)) { + output = fs.createSyncable(path, 0, replication, blockSize); + } else { + output = fs.create(path, true, 0, replication, blockSize); + } + try { + output.setDropBehind(Boolean.TRUE); + } catch (UnsupportedOperationException e) { + // Not all filesystems support drop-behind. + } catch (IOException e) { + // Drop-behind is a performance hint, not a requirement. + } + if (!fs.canSyncAndFlush(path)) { + org.slf4j.LoggerFactory.getLogger(DfsWalProvider.class) + .warn("sync not supported for log file {}. Data loss may occur.", logPath); + } + return new DfsWriter(logPath, output); + } + + @Override + public InputStream open(String reference) throws IOException { + return context.getVolumeManager().open(new Path(reference)); + } + + @Override + public boolean exists(String reference) throws IOException { + return context.getVolumeManager().exists(new Path(reference)); + } + + @Override + public void delete(String reference) throws IOException { + VolumeManager fs = context.getVolumeManager(); + Path path = new Path(reference); + if (!fs.moveToTrash(path)) { + fs.deleteRecursively(path); + } + } + + private static class DfsWriter implements Writer { + private final String reference; + private final FSDataOutputStream output; + private boolean closed; + + DfsWriter(String reference, FSDataOutputStream output) { + this.reference = reference; + this.output = output; + } + + @Override + public String getReference() { + return reference; + } + + @Override + public java.io.OutputStream getOutputStream() { + return output; + } + + @Override + public void sync(Durability durability) throws IOException { + if (durability == Durability.SYNC) { + output.hsync(); + } else if (durability == Durability.FLUSH) { + output.hflush(); + } else { + throw new IllegalArgumentException("Unexpected WAL sync durability " + durability); + } + } + + @Override + public long getLength() throws IOException { + return output.getPos(); + } + + @Override + public int getCurrentReplication() throws IOException { + var wrapped = output.getWrappedStream(); + return wrapped instanceof DFSOutputStream dfs ? dfs.getCurrentBlockReplication() : -1; + } + + @Override + public String[] getPipeline() { + var wrapped = output.getWrappedStream(); + if (wrapped instanceof DFSOutputStream dfs) { + DatanodeInfo[] pipeline = dfs.getPipeline(); + String[] names = new String[pipeline.length]; + for (int i = 0; i < pipeline.length; i++) { + names[i] = pipeline[i].getName(); + } + return names; + } + return new String[0]; + } + + @Override + public synchronized void close() throws IOException { + if (!closed) { + output.close(); + closed = true; + } + } + } +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java new file mode 100644 index 00000000000..e08f4970100 --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.log; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.accumulo.core.classloader.ClassLoaderUtil; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.PropertyType; +import org.apache.accumulo.core.spi.common.ServiceEnvironment; +import org.apache.accumulo.core.spi.wal.WalProvider; +import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.ServiceEnvironmentImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** Loads and validates the configured WAL providers for a server process. */ +public class WalProviderRegistry { + + private static final Logger log = LoggerFactory.getLogger(WalProviderRegistry.class); + private static final ObjectMapper JSON_MAPPER = + new ObjectMapper().enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY) + .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); + + private final Map providers = new HashMap<>(); + private final Map providersByClass = new HashMap<>(); + private final WalProvider current; + + static void validateProviderId(String id, String className) { + if (id == null || !id.matches(WalProvider.PROVIDER_ID_REGEX)) { + throw new IllegalArgumentException("Invalid WAL provider ID '" + id + "' from " + className + + "; IDs must match " + WalProvider.PROVIDER_ID_REGEX); + } + } + + public WalProviderRegistry(ServerContext context) { + var conf = context.getConfiguration(); + ClassLoaderUtil.initContextFactory(conf); + var environment = new ServiceEnvironmentImpl(context); + String currentClass = conf.get(Property.INSTANCE_WAL_PROVIDER).trim(); + if (currentClass.isEmpty()) { + throw new IllegalArgumentException( + "No current WAL provider configured in " + Property.INSTANCE_WAL_PROVIDER); + } + var providerOptions = parseProviderConfig(conf.get(Property.INSTANCE_WAL_PROVIDERS)); + for (var entry : providerOptions.entrySet()) { + String className = entry.getKey(); + try { + WalProvider provider = + ConfigurationTypeHelper.getClassInstance(null, className, WalProvider.class); + provider.init(new ProviderInitParameters(environment, entry.getValue())); + String id = provider.getId(); + validateProviderId(id, className); + if (id.equals(DfsWalProvider.PROVIDER_ID) + && !className.equals(DfsWalProvider.class.getName())) { + throw new IllegalArgumentException("WAL provider ID '" + DfsWalProvider.PROVIDER_ID + + "' is reserved for legacy filesystem WALs and " + DfsWalProvider.class.getName()); + } + WalProvider previous = providers.putIfAbsent(id, provider); + if (previous != null) { + throw new IllegalArgumentException("Duplicate WAL provider ID '" + id + "' from " + + previous.getClass().getName() + " and " + className); + } + providersByClass.put(className, provider); + log.info("Loaded WAL provider {} with ID {}", className, id); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Unable to load configured WAL provider " + className, e); + } + } + + current = providersByClass.get(currentClass); + if (current == null) { + throw new IllegalArgumentException( + "Current WAL provider " + currentClass + " from " + Property.INSTANCE_WAL_PROVIDER + + " is not listed in " + Property.INSTANCE_WAL_PROVIDERS); + } + } + + static Map> parseProviderConfig(String json) { + final JsonNode root; + try { + root = JSON_MAPPER.readTree(json); + } catch (IOException e) { + throw new IllegalArgumentException("Invalid JSON in " + Property.INSTANCE_WAL_PROVIDERS, e); + } + if (root == null || !root.isObject()) { + throw new IllegalArgumentException(Property.INSTANCE_WAL_PROVIDERS + + " must be a JSON object mapping provider classes to option maps"); + } + + Map> providers = new LinkedHashMap<>(); + root.fields().forEachRemaining(entry -> { + String className = entry.getKey(); + if (className.isBlank() || !PropertyType.CLASSNAME.isValidFormat(className)) { + throw new IllegalArgumentException("Invalid WAL provider class name '" + className + "'"); + } + JsonNode optionsNode = entry.getValue(); + if (!optionsNode.isObject()) { + throw new IllegalArgumentException("Options for WAL provider " + className + + " must be a JSON object (use {} when there are no options)"); + } + Map options = new LinkedHashMap<>(); + optionsNode.fields().forEachRemaining(option -> { + if (!option.getValue().isTextual()) { + throw new IllegalArgumentException("Option '" + option.getKey() + "' for WAL provider " + + className + " must have a string value"); + } + options.put(option.getKey(), option.getValue().textValue()); + }); + providers.put(className, Collections.unmodifiableMap(options)); + }); + return Collections.unmodifiableMap(providers); + } + + private record ProviderInitParameters(ServiceEnvironment environment, Map options) + implements WalProvider.InitParameters { + @Override + public ServiceEnvironment getServiceEnvironment() { + return environment; + } + + @Override + public Map getOptions() { + return options; + } + } + + public WalProvider getCurrent() { + return current; + } + + public WalProvider get(String id) throws ProviderNotFoundException { + WalProvider provider = providers.get(id); + if (provider == null) { + throw new ProviderNotFoundException(id); + } + return provider; + } + + public Set getIds() { + return Set.copyOf(providers.keySet()); + } + + public static class ProviderNotFoundException extends IOException { + private static final long serialVersionUID = 1L; + + public ProviderNotFoundException(String id) { + super("WAL provider with ID '" + id + "' is not configured in " + + Property.INSTANCE_WAL_PROVIDERS); + } + } +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java index a0c18971a39..eaca6c4ebd8 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java @@ -33,6 +33,7 @@ import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeExistsPolicy; import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeMissingPolicy; import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.server.ServerContext; import org.apache.hadoop.fs.Path; import org.apache.zookeeper.KeeperException; @@ -51,7 +52,7 @@ * The state of the WALs is kept in Zookeeper under /accumulo/<instanceid>/wals. For each * server, there is a znode formatted like the TServerInstance.toString(): "host:port[sessionid]". * Under the server znode, is a node for each log, using the UUID for the log. In each of the WAL - * znodes, is the current state of the log, and the full path to the log. + * znodes, is the current state of the log, the provider ID, and the full path to the log. * *

* The state [OPEN, CLOSED, UNREFERENCED] is what the tablet server believes to be the state of the @@ -86,7 +87,7 @@ public enum WalState { UNREFERENCED } - public record WalStatePath(WalState state, Path path) { + public record WalStatePath(WalState state, String providerId, Path path) { } private final ZooReaderWriter zoo; @@ -131,12 +132,21 @@ public void initWalMarker(TServerInstance tsi) throws WalMarkerException { // Tablet server opens a new WAL public void addNewWalMarker(TServerInstance tsi, Path path) throws WalMarkerException { - updateState(tsi, path, WalState.OPEN); + updateState(tsi, DfsWalProvider.PROVIDER_ID, path, WalState.OPEN); + } + + public void addNewWalMarker(TServerInstance tsi, LogEntry entry) throws WalMarkerException { + updateState(tsi, entry.getProviderId(), new Path(entry.getPath()), WalState.OPEN); } private void updateState(TServerInstance tsi, Path path, WalState state) throws WalMarkerException { - byte[] data = (state + "," + path).getBytes(UTF_8); + updateState(tsi, DfsWalProvider.PROVIDER_ID, path, state); + } + + private void updateState(TServerInstance tsi, String providerId, Path path, WalState state) + throws WalMarkerException { + byte[] data = (state + "," + providerId + "," + path).getBytes(UTF_8); try { NodeExistsPolicy policy = NodeExistsPolicy.OVERWRITE; if (state == WalState.OPEN) { @@ -157,14 +167,25 @@ public void walUnreferenced(TServerInstance tsi, Path path) throws WalMarkerExce updateState(tsi, path, WalState.UNREFERENCED); } + public void walUnreferenced(TServerInstance tsi, LogEntry entry) throws WalMarkerException { + updateState(tsi, entry.getProviderId(), new Path(entry.getPath()), WalState.UNREFERENCED); + } + + public void closeWal(TServerInstance tsi, LogEntry entry) throws WalMarkerException { + closeWal(tsi, new Path(entry.getPath()), entry.getProviderId()); + } + public static WalStatePath parse(byte[] data) { - String[] parts = new String(data, UTF_8).split(","); - return new WalStatePath(WalState.valueOf(parts[0]), new Path(parts[1])); + String[] parts = new String(data, UTF_8).split(",", 3); + if (parts.length != 3) { + throw new IllegalArgumentException("Malformed WAL state: " + new String(data, UTF_8)); + } + return new WalStatePath(WalState.valueOf(parts[0]), parts[1], new Path(parts[2])); } // Manager needs to know the logs for the given instance - public List getWalsInUse(TServerInstance tsi) throws WalMarkerException { - List result = new ArrayList<>(); + public List getWalsInUse(TServerInstance tsi) throws WalMarkerException { + List result = new ArrayList<>(); try { String zpath = root() + "/" + tsi; zoo.sync(zpath); @@ -183,7 +204,7 @@ public List getWalsInUse(TServerInstance tsi) throws WalMarkerException { if (zdata != null) { WalStatePath parts = parse(zdata); if (parts.state() != WalState.UNREFERENCED) { - result.add(parts.path()); + result.add(LogEntry.fromPath(parts.providerId(), parts.path().toString())); } } } @@ -244,7 +265,13 @@ public Set getAllState() throws WalMarkerException { for (Entry> entry : markers.entrySet()) { for (UUID id : entry.getValue()) { // WAL markers can be deleted while reading them. - result.add(state(entry.getKey(), id)); + try { + result.add(state(entry.getKey(), id)); + } catch (WalMarkerException e) { + if (!(e.getCause() instanceof KeeperException.NoNodeException)) { + throw e; + } + } } } return result; @@ -281,6 +308,21 @@ public void forget(TServerInstance instance) throws WalMarkerException { // manager can mark a log as unreferenced after it has made log recovery markers on the tablets // that need to be recovered public void closeWal(TServerInstance instance, Path path) throws WalMarkerException { - updateState(instance, path, WalState.CLOSED); + closeWal(instance, path, null); + } + + private void closeWal(TServerInstance instance, Path path, String providerId) + throws WalMarkerException { + WalStatePath existing; + try { + existing = state(instance, UUID.fromString(path.getName())); + } catch (WalMarkerException e) { + if (e.getCause() instanceof KeeperException.NoNodeException) { + return; + } + throw e; + } + updateState(instance, providerId == null ? existing.providerId() : providerId, path, + WalState.CLOSED); } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java index 775331960a3..a550c43cc9f 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/AbstractTabletStateStore.java @@ -36,7 +36,6 @@ import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.server.ServerContext; -import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -114,13 +113,13 @@ public Set setFutureLocations(Collection assignments) @Override public void unassign(Collection tablets, - Map> logsForDeadServers) throws DistributedStoreException { + Map> logsForDeadServers) throws DistributedStoreException { unassign(tablets, logsForDeadServers, null); } @Override public void suspend(Collection tablets, - Map> logsForDeadServers, SteadyTime suspensionTimestamp) + Map> logsForDeadServers, SteadyTime suspensionTimestamp) throws DistributedStoreException { unassign(tablets, logsForDeadServers, suspensionTimestamp); } @@ -129,7 +128,7 @@ protected abstract void processSuspension(Ample.ConditionalTabletMutator tabletM TabletMetadata tm, SteadyTime suspensionTimestamp); private void unassign(Collection tablets, - Map> logsForDeadServers, SteadyTime suspensionTimestamp) + Map> logsForDeadServers, SteadyTime suspensionTimestamp) throws DistributedStoreException { try (var tabletsMutator = ample.conditionallyMutateTablets()) { for (TabletMetadata tm : tablets) { @@ -145,11 +144,10 @@ private void unassign(Collection tablets, updateLastLocation(tabletMutator, tm.getLocation().getServerInstance(), tm.getLast()); tabletMutator.deleteLocation(tm.getLocation()); if (logsForDeadServers != null) { - List logs = logsForDeadServers.get(tm.getLocation().getServerInstance()); + List logs = logsForDeadServers.get(tm.getLocation().getServerInstance()); if (logs != null) { - for (Path log : logs) { - LogEntry entry = LogEntry.fromPath(log.toString()); - tabletMutator.putWal(entry); + for (LogEntry log : logs) { + tabletMutator.putWal(log); } } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java index b5da85b3ad9..bd1cd462604 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java @@ -30,8 +30,8 @@ import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.time.SteadyTime; -import org.apache.hadoop.fs.Path; import com.google.common.net.HostAndPort; @@ -82,7 +82,7 @@ public void setLocations(Collection assignments) throws DistributedS @Override public void unassign(Collection tablets, - Map> logsForDeadServers) throws DistributedStoreException { + Map> logsForDeadServers) throws DistributedStoreException { wrapped.unassign(tablets, logsForDeadServers); if (logsForDeadServers == null) { @@ -96,7 +96,7 @@ public void unassign(Collection tablets, @Override public void suspend(Collection tablets, - Map> logsForDeadServers, SteadyTime suspensionTimestamp) + Map> logsForDeadServers, SteadyTime suspensionTimestamp) throws DistributedStoreException { wrapped.suspend(tablets, logsForDeadServers, suspensionTimestamp); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java index d369f7bae76..06eefd79892 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java @@ -31,9 +31,9 @@ import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.server.ServerContext; -import org.apache.hadoop.fs.Path; /** * Interface for storing information about tablet assignments. There are three implementations: @@ -84,14 +84,14 @@ Set setFutureLocations(Collection assignments) * @param logsForDeadServers a cache of logs in use by servers when they died */ void unassign(Collection tablets, - Map> logsForDeadServers) throws DistributedStoreException; + Map> logsForDeadServers) throws DistributedStoreException; /** * Mark tablets as having no known or future location, but desiring to be returned to their * previous tserver. */ void suspend(Collection tablets, - Map> logsForDeadServers, SteadyTime suspensionTimestamp) + Map> logsForDeadServers, SteadyTime suspensionTimestamp) throws DistributedStoreException; /** @@ -100,13 +100,13 @@ void suspend(Collection tablets, void unsuspend(Collection tablets) throws DistributedStoreException; public static void unassign(ServerContext context, TabletMetadata tm, - Map> logsForDeadServers) throws DistributedStoreException { + Map> logsForDeadServers) throws DistributedStoreException { getStoreForTablet(tm.getExtent(), context).unassign(Collections.singletonList(tm), logsForDeadServers); } public static void suspend(ServerContext context, TabletMetadata tm, - Map> logsForDeadServers, SteadyTime suspensionTimestamp) + Map> logsForDeadServers, SteadyTime suspensionTimestamp) throws DistributedStoreException { getStoreForTablet(tm.getExtent(), context).suspend(Collections.singletonList(tm), logsForDeadServers, suspensionTimestamp); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java index 600b96a8feb..92f01ca8c6b 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java @@ -43,11 +43,11 @@ import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.RootTabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.iterators.SystemIteratorEnvironment; import org.apache.accumulo.server.iterators.SystemIteratorEnvironmentImpl; -import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -177,7 +177,7 @@ private static void validateTablets(Collection tablets) { @Override public void unassign(Collection tablets, - Map> logsForDeadServers) throws DistributedStoreException { + Map> logsForDeadServers) throws DistributedStoreException { validateTablets(tablets); super.unassign(tablets, logsForDeadServers); @@ -187,7 +187,7 @@ public void unassign(Collection tablets, @Override public void suspend(Collection tablets, - Map> logsForDeadServers, SteadyTime suspensionTimestamp) + Map> logsForDeadServers, SteadyTime suspensionTimestamp) throws DistributedStoreException { validateTablets(tablets); super.suspend(tablets, logsForDeadServers, suspensionTimestamp); diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java index 0c0998812cf..1c7c3c8288e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/checkCommand/SystemConfigCheckRunner.java @@ -171,7 +171,7 @@ private static boolean checkZKWALsMetadata(ServerContext context) throws Excepti Map> missingWals = new HashMap<>(); for (var instanceAndWals : walsBefore.entrySet()) { for (var wal : instanceAndWals.getValue()) { - if (!context.getVolumeManager().exists(wal.path())) { + if (!context.getWalProviderRegistry().get(wal.providerId()).exists(wal.path().toString())) { missingWals.computeIfAbsent(instanceAndWals.getKey(), k -> new HashSet<>()).add(wal); } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java index 033cf172125..46d65958f5e 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java @@ -27,6 +27,7 @@ import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.server.fs.VolumeManager.FileType; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.Test; @@ -170,17 +171,17 @@ public void testWalVolumeReplacment() { String walUUID = UUID.randomUUID().toString(); String fileName = "hdfs://nn1/accumulo/wal/localhost+9997/" + walUUID; - LogEntry le = LogEntry.fromPath(fileName); + LogEntry le = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, fileName); LogEntry fixedVolume = VolumeUtil.switchVolume(le, replacements); assertEquals("viewfs:/a/accumulo/wal/localhost+9997/" + walUUID, fixedVolume.getPath()); fileName = "hdfs://nn1:9000/accumulo/wal/localhost+9997/" + walUUID; - le = LogEntry.fromPath(fileName); + le = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, fileName); fixedVolume = VolumeUtil.switchVolume(le, replacements); assertEquals("viewfs:/a/accumulo/wal/localhost+9997/" + walUUID, fixedVolume.getPath()); fileName = "hdfs://nn2/accumulo/wal/localhost+9997/" + walUUID; - le = LogEntry.fromPath(fileName); + le = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, fileName); fixedVolume = VolumeUtil.switchVolume(le, replacements); assertEquals("viewfs:/b/accumulo/wal/localhost+9997/" + walUUID, fixedVolume.getPath()); } diff --git a/server/base/src/test/java/org/apache/accumulo/server/log/WalProviderRegistryTest.java b/server/base/src/test/java/org/apache/accumulo/server/log/WalProviderRegistryTest.java new file mode 100644 index 00000000000..b50feaa7b06 --- /dev/null +++ b/server/base/src/test/java/org/apache/accumulo/server/log/WalProviderRegistryTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.log; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +class WalProviderRegistryTest { + + @Test + void providerIdsMustBeAlphanumeric() { + assertDoesNotThrow(() -> WalProviderRegistry.validateProviderId("Provider123", "provider")); + assertThrows(IllegalArgumentException.class, + () -> WalProviderRegistry.validateProviderId("provider-id", "provider")); + assertThrows(IllegalArgumentException.class, + () -> WalProviderRegistry.validateProviderId("provider_id", "provider")); + assertThrows(IllegalArgumentException.class, + () -> WalProviderRegistry.validateProviderId("provider.id", "provider")); + assertThrows(IllegalArgumentException.class, + () -> WalProviderRegistry.validateProviderId("-", "provider")); + } + + @Test + void parsesProviderClassesAndOptions() { + var config = WalProviderRegistry.parseProviderConfig( + "{\"org.example.Provider\":{},\"org.example.OtherProvider\":{\"endpoint\":\"test\",\"retries\":\"3\"}}"); + + assertEquals(Map.of("org.example.Provider", Map.of(), "org.example.OtherProvider", + Map.of("endpoint", "test", "retries", "3")), config); + } + + @Test + void rejectsInvalidProviderConfiguration() { + List.of("[]", "\"provider\"", "{\"\":{}}", "{\"org.example.Provider\":\"\"}", + "{\"org.example.Provider\":{\"count\":3}}", "{\"org.example.Provider\":{}} trailing", + "{\"org.example.Provider\":{},\"org.example.Provider\":{}}") + .forEach(json -> assertThrows(IllegalArgumentException.class, + () -> WalProviderRegistry.parseProviderConfig(json), json)); + } +} diff --git a/server/base/src/test/java/org/apache/accumulo/server/log/WalStateManagerTest.java b/server/base/src/test/java/org/apache/accumulo/server/log/WalStateManagerTest.java new file mode 100644 index 00000000000..90919873aae --- /dev/null +++ b/server/base/src/test/java/org/apache/accumulo/server/log/WalStateManagerTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.log; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; + +import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter; +import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.tabletserver.log.LogEntry; +import org.apache.accumulo.core.zookeeper.ZooSession; +import org.apache.accumulo.server.ServerContext; +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.Test; + +class WalStateManagerTest { + + @Test + void parseDefaultProviderWalState() { + var parsed = WalStateManager + .parse(("CLOSED," + DfsWalProvider.PROVIDER_ID + ",hdfs://nn/accumulo/wal/host+1234/id") + .getBytes(UTF_8)); + assertEquals(WalStateManager.WalState.CLOSED, parsed.state()); + assertEquals(DfsWalProvider.PROVIDER_ID, parsed.providerId()); + assertEquals(new Path("hdfs://nn/accumulo/wal/host+1234/id"), parsed.path()); + } + + @Test + void rejectLegacyWalState() { + assertThrows(IllegalArgumentException.class, + () -> WalStateManager.parse("CLOSED,hdfs://nn/accumulo/wal/host+1234/id".getBytes(UTF_8))); + } + + @Test + void parseProviderWalState() { + var parsed = WalStateManager + .parse("UNREFERENCED,example,hdfs://nn/accumulo/wal/host+1234/id".getBytes(UTF_8)); + assertEquals(WalStateManager.WalState.UNREFERENCED, parsed.state()); + assertEquals("example", parsed.providerId()); + assertEquals(new Path("hdfs://nn/accumulo/wal/host+1234/id"), parsed.path()); + } + + @Test + void getWalsInUsePreservesProviderId() throws Exception { + ServerContext context = createMock(ServerContext.class); + ZooSession session = createMock(ZooSession.class); + ZooReaderWriter zoo = createMock(ZooReaderWriter.class); + TServerInstance server = new TServerInstance("localhost:1234[SESSION]"); + String serverPath = WalStateManager.ZWALS + "/" + server; + String reference = + "customfs://nn/accumulo/wal/localhost+1234/11111111-1111-1111-1111-111111111111"; + + expect(context.getZooSession()).andReturn(session); + expect(session.asReaderWriter()).andReturn(zoo); + expect(zoo.exists(WalStateManager.ZWALS)).andReturn(true); + zoo.sync(serverPath); + expectLastCall(); + expect(zoo.getChildren(serverPath)).andReturn(List.of("11111111-1111-1111-1111-111111111111")); + expect(zoo.getData(serverPath + "/11111111-1111-1111-1111-111111111111")) + .andReturn(("CLOSED,custom," + reference).getBytes(UTF_8)); + + replay(context, session, zoo); + WalStateManager manager = new WalStateManager(context); + assertEquals(List.of(LogEntry.fromPath("custom", reference)), manager.getWalsInUse(server)); + verify(context, session, zoo); + } +} diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java index 0bab62dfbc2..21d5fa0b1c1 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java @@ -55,6 +55,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.tabletserver.log.LogEntry; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; @@ -108,9 +109,11 @@ private SortedMap createMetadataEntryKV(KeyExtent extent) { mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555"); - LogEntry le1 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + LogEntry le1 = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); le1.addToMutation(mutation); - LogEntry le2 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + LogEntry le2 = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); le2.addToMutation(mutation); StoredTabletFile sf1 = diff --git a/server/base/src/test/resources/accumulo.properties b/server/base/src/test/resources/accumulo.properties index b63ed5c0617..2a7b4af5260 100644 --- a/server/base/src/test/resources/accumulo.properties +++ b/server/base/src/test/resources/accumulo.properties @@ -21,3 +21,5 @@ instance.dfs.dir=${project.build.directory}/instanceTest instance.secret=TEST_SYSTEM_SECRET +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java index b09c9ea9975..6146b1d2147 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java @@ -56,6 +56,7 @@ import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.WalStateManager; import org.apache.accumulo.server.log.WalStateManager.WalMarkerException; import org.apache.accumulo.server.log.WalStateManager.WalState; @@ -270,8 +271,23 @@ private long removeFiles(Collection collection, final GCStatus sta try { for (WalStatePath stateFile : collection) { Path path = stateFile.path(); - futures.put(path, removeFile(deleteThreadPool, path, counter, - "Removing " + stateFile.state() + " WAL " + path)); + if (stateFile.providerId().equals(DfsWalProvider.PROVIDER_ID)) { + futures.put(path, removeFile(deleteThreadPool, path, counter, + "Removing " + stateFile.state() + " WAL " + path)); + } else { + futures.put(path, deleteThreadPool.submit(() -> { + try { + log.debug("Removing {} WAL {} using provider {}", stateFile.state(), path, + stateFile.providerId()); + context.getWalProviderRegistry().get(stateFile.providerId()).delete(path.toString()); + counter.incrementAndGet(); + } catch (FileNotFoundException e) { + // ignored + } catch (IOException e) { + log.error("Unable to delete {} WAL {}", stateFile.providerId(), path, e); + } + })); + } } while (!futures.isEmpty()) { diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 13044b69aa9..516897970fc 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@ -96,6 +96,7 @@ public class SimpleGarbageCollector extends AbstractServer implements Iface { protected SimpleGarbageCollector(ServerOpts opts, String[] args) { super(ServerId.Type.GARBAGE_COLLECTOR, opts, ServerContext::new, args); + getContext().getWalProviderRegistry(); final AccumuloConfiguration conf = getConfiguration(); diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java index 99064550840..e769e104562 100644 --- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java +++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java @@ -41,6 +41,7 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.WalStateManager; import org.apache.accumulo.server.log.WalStateManager.WalState; import org.apache.accumulo.server.log.WalStateManager.WalStatePath; @@ -99,7 +100,7 @@ public void testRemoveUnusedLog() throws Exception { EasyMock.expect(marker.getAllMarkers()).andReturn(markers).once(); EasyMock.expect(marker.state(server1, id)) - .andReturn(new WalStatePath(WalState.UNREFERENCED, path)); + .andReturn(new WalStatePath(WalState.UNREFERENCED, DfsWalProvider.PROVIDER_ID, path)); EasyMock.expect(fs.deleteRecursively(path)).andReturn(true).once(); marker.removeWalMarker(server1, id); EasyMock.expectLastCall().once(); @@ -139,7 +140,8 @@ public void testKeepClosedLog() throws Exception { EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1)); EasyMock.expect(marker.getAllMarkers()).andReturn(markers).once(); - EasyMock.expect(marker.state(server1, id)).andReturn(new WalStatePath(WalState.CLOSED, path)); + EasyMock.expect(marker.state(server1, id)) + .andReturn(new WalStatePath(WalState.CLOSED, DfsWalProvider.PROVIDER_ID, path)); EasyMock.replay(conf, context, marker, tserverSet, fs); var gc = new GarbageCollectWriteAheadLogs(context, fs, tserverSet, marker) { @@ -175,7 +177,8 @@ public void deleteUnreferencedLogOnDeadServer() throws Exception { EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1)); EasyMock.expect(marker.getAllMarkers()).andReturn(markers2).once(); - EasyMock.expect(marker.state(server2, id)).andReturn(new WalStatePath(WalState.OPEN, path)); + EasyMock.expect(marker.state(server2, id)) + .andReturn(new WalStatePath(WalState.OPEN, DfsWalProvider.PROVIDER_ID, path)); EasyMock.expect(fs.deleteRecursively(path)).andReturn(true).once(); marker.removeWalMarker(server2, id); @@ -215,7 +218,8 @@ public void ignoreReferenceLogOnDeadServer() throws Exception { EasyMock.expect(tserverSet.getCurrentServers()).andReturn(Collections.singleton(server1)); EasyMock.expect(marker.getAllMarkers()).andReturn(markers2).once(); - EasyMock.expect(marker.state(server2, id)).andReturn(new WalStatePath(WalState.OPEN, path)); + EasyMock.expect(marker.state(server2, id)) + .andReturn(new WalStatePath(WalState.OPEN, DfsWalProvider.PROVIDER_ID, path)); EasyMock.replay(conf, context, fs, marker, tserverSet); GarbageCollectWriteAheadLogs gc = diff --git a/server/gc/src/test/resources/accumulo.properties b/server/gc/src/test/resources/accumulo.properties index 7a9d5c90663..1669b68ee68 100644 --- a/server/gc/src/test/resources/accumulo.properties +++ b/server/gc/src/test/resources/accumulo.properties @@ -18,3 +18,5 @@ # # File needs to exist for unit tests +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/BalanceManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/BalanceManager.java index 9baae8d74dd..5135d72965c 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/BalanceManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/BalanceManager.java @@ -312,7 +312,7 @@ private void balanceCompleted(Ample.DataLevel level, long migrations) { /** * Waits for the given data levels to complete a run of balancing with zero migrations. */ - private void waitForBalance(Set levels) { + void waitForBalance(Set levels) { synchronized (balancedNotifier) { var snapshot = new EnumMap(Ample.DataLevel.class); for (var dl : levels) { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index ed20d3e0ffc..ce0ac1e9dbb 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -104,8 +105,10 @@ import org.apache.accumulo.core.metadata.SystemTables; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metrics.MetricsInfo; import org.apache.accumulo.core.metrics.MetricsProducer; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.core.util.Retry; import org.apache.accumulo.core.util.Timer; @@ -133,6 +136,9 @@ import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.compaction.CompactionConfigStorage; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.log.WalProviderRegistry; +import org.apache.accumulo.server.log.WalStateManager; +import org.apache.accumulo.server.log.WalStateManager.WalStatePath; import org.apache.accumulo.server.manager.LiveTServerSet; import org.apache.accumulo.server.manager.LiveTServerSet.LiveTServersSnapshot; import org.apache.accumulo.server.manager.LiveTServerSet.TServerConnection; @@ -486,6 +492,7 @@ protected Manager(ServerOpts opts, String[] args) throws IOException { super(ServerId.Type.MANAGER, opts, serverContextFactory, args); ServerContext context = super.getContext(); + context.getWalProviderRegistry(); upgradeCoordinator = new UpgradeCoordinator(context); balanceManager = new BalanceManager(); @@ -917,6 +924,36 @@ private void setupPrimaryMetrics() { } } + private void validateWalProviderReferences() { + getBalanceManager().waitForBalance(EnumSet.of(DataLevel.ROOT, DataLevel.METADATA)); + WalProviderRegistry registry = getContext().getWalProviderRegistry(); + Set referencedIds = new HashSet<>(); + for (DataLevel level : DataLevel.values()) { + try (var tablets = getContext().getAmple().readTablets().forLevel(level) + .fetch(TabletMetadata.ColumnType.LOGS).build()) { + for (TabletMetadata tablet : tablets) { + for (LogEntry entry : tablet.getLogs()) { + referencedIds.add(entry.getProviderId()); + } + } + } + } + try { + for (WalStatePath state : new WalStateManager(getContext()).getAllState()) { + referencedIds.add(state.providerId()); + } + } catch (WalStateManager.WalMarkerException e) { + throw new IllegalStateException("Unable to audit WAL provider markers", e); + } + referencedIds.removeAll(registry.getIds()); + if (!referencedIds.isEmpty()) { + throw new IllegalStateException("WAL provider IDs referenced by tablet metadata or WAL " + + "markers are not configured in " + Property.INSTANCE_WAL_PROVIDERS + ": " + + referencedIds); + } + log.info("Validated configured WAL providers against all tablet metadata and WAL markers"); + } + @Override public void run() { final ServerContext context = getContext(); @@ -1208,6 +1245,11 @@ boolean canSuspendTablets() { log.info("AuthenticationTokenSecretManager is initialized"); } + // Validate that the wals in the tables use + // known providers. If that's not the case, + // then abort. + validateWalProviderReferences(); + // Replace the ServiceLockData information in the Manager lock node in ZooKeeper. // This advertises the address that clients can use to connect to the Manager // for the Coordinator, Fate, and Manager services. Do **not** do this until diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java index fb2eaa62a9f..1719198a305 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java @@ -86,6 +86,7 @@ import org.apache.accumulo.core.metrics.Metric; import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.Timer; import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.core.util.threads.Threads.AccumuloDaemonThread; @@ -111,7 +112,6 @@ import org.apache.accumulo.server.manager.state.TabletManagementParameters; import org.apache.accumulo.server.manager.state.TabletStateStore; import org.apache.accumulo.server.manager.state.UnassignedTablet; -import org.apache.hadoop.fs.Path; import org.apache.thrift.TException; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -257,7 +257,7 @@ private static class TabletLists { private final List assignedToDeadServers = new ArrayList<>(); private final List suspendedToGoneServers = new ArrayList<>(); private final Map unassigned = new HashMap<>(); - private final Map> logsForDeadServers = new TreeMap<>(); + private final Map> logsForDeadServers = new TreeMap<>(); // read only list of tablet servers that are not shutting down private final SortedMap destinations; private final Map> currentTServerGrouping; @@ -1206,10 +1206,10 @@ private void replaceVolumes(List volumeReplacemen } private static void markDeadServerLogsAsClosed(WalStateManager mgr, - Map> logsForDeadServers) throws WalMarkerException { - for (Entry> server : logsForDeadServers.entrySet()) { - for (Path path : server.getValue()) { - mgr.closeWal(server.getKey(), path); + Map> logsForDeadServers) throws WalMarkerException { + for (Entry> server : logsForDeadServers.entrySet()) { + for (LogEntry log : server.getValue()) { + mgr.closeWal(server.getKey(), log); } } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java index 8e1e00988f6..f1f7dc63b0e 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java @@ -41,6 +41,7 @@ import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.manager.Manager; import org.apache.accumulo.server.fs.VolumeUtil; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.SortedLogState; import org.apache.accumulo.server.manager.recovery.HadoopLogCloser; import org.apache.accumulo.server.manager.recovery.LogCloser; @@ -87,10 +88,13 @@ private class LogSortTask implements Runnable { private final String source; private final String destination; private final String sortId; + private final String providerId; private final LogCloser closer; - public LogSortTask(LogCloser closer, String source, String destination, String sortId) { + public LogSortTask(LogCloser closer, String providerId, String source, String destination, + String sortId) { this.closer = closer; + this.providerId = providerId; this.source = source; this.destination = destination; this.sortId = sortId; @@ -100,15 +104,15 @@ public LogSortTask(LogCloser closer, String source, String destination, String s public void run() { boolean rescheduled = false; try { - long time = closer.close(manager.getConfiguration(), manager.getContext().getHadoopConf(), - manager.getVolumeManager(), new Path(source)); + long time = closer == null ? 0 : closer.close(manager.getConfiguration(), + manager.getContext().getHadoopConf(), manager.getVolumeManager(), new Path(source)); if (time > 0) { ScheduledFuture future = executor.schedule(this, time, TimeUnit.MILLISECONDS); ThreadPools.watchNonCriticalScheduledTask(future); rescheduled = true; } else { - initiateSort(sortId, source, destination); + initiateSort(sortId, providerId, source, destination); } } catch (FileNotFoundException e) { log.debug("Unable to initiate log sort for " + source + ": " + e); @@ -125,9 +129,9 @@ public void run() { } - private void initiateSort(String sortId, String source, final String destination) - throws KeeperException, InterruptedException { - String work = source + "|" + destination; + private void initiateSort(String sortId, String providerId, String source, + final String destination) throws KeeperException, InterruptedException { + String work = providerId + "|" + source + "|" + destination; new DistributedWorkQueue(Constants.ZRECOVERY, manager.getConfiguration(), manager) .addWork(sortId, work.getBytes(UTF_8)); @@ -180,8 +184,8 @@ public RecoverySession newRecoverySession() { private boolean recoverLog(LogEntry walog) throws IOException { boolean recoveryNeeded = false; - LogEntry switchedWalog = - VolumeUtil.switchVolume(walog, manager.getContext().getVolumeReplacements()); + LogEntry switchedWalog = walog.getProviderId().equals(DfsWalProvider.PROVIDER_ID) + ? VolumeUtil.switchVolume(walog, manager.getContext().getVolumeReplacements()) : null; if (switchedWalog != null) { // replaces the volume used for sorting, but do not change entry in metadata table. When // the tablet loads it will change the metadata table entry. If @@ -221,8 +225,10 @@ private boolean recoverLog(LogEntry walog) throws IOException { synchronized (this) { if (!closeTasksQueued.contains(sortId) && !sortsQueued.contains(sortId)) { AccumuloConfiguration aconf = manager.getConfiguration(); - LogCloser closer = Property.createInstanceFromPropertyName(aconf, - Property.MANAGER_WAL_CLOSER_IMPLEMENTATION, LogCloser.class, new HadoopLogCloser()); + LogCloser closer = walog.getProviderId().equals(DfsWalProvider.PROVIDER_ID) + ? Property.createInstanceFromPropertyName(aconf, + Property.MANAGER_WAL_CLOSER_IMPLEMENTATION, LogCloser.class, new HadoopLogCloser()) + : null; Long delay = recoveryDelay.get(sortId); if (delay == null) { delay = aconf.getTimeInMillis(Property.MANAGER_RECOVERY_DELAY); @@ -233,7 +239,8 @@ private boolean recoverLog(LogEntry walog) throws IOException { log.info("Starting recovery of {} (in : {}s)", filename, (delay / 1000)); ScheduledFuture future = executor.schedule( - new LogSortTask(closer, filename, dest, sortId), delay, TimeUnit.MILLISECONDS); + new LogSortTask(closer, walog.getProviderId(), filename, dest, sortId), delay, + TimeUnit.MILLISECONDS); ThreadPools.watchNonCriticalScheduledTask(future); closeTasksQueued.add(sortId); recoveryDelay.put(sortId, delay); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java index d4ea54cc55b..43483578d5e 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java @@ -80,6 +80,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily; import org.apache.accumulo.core.metadata.schema.RootTabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMergeabilityMetadata; @@ -87,6 +88,7 @@ import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.core.schema.Section; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.spi.wal.WalProvider; import org.apache.accumulo.core.util.Encoding; import org.apache.accumulo.core.util.TextUtil; import org.apache.accumulo.core.util.UtilWaitThread; @@ -101,6 +103,8 @@ import org.apache.accumulo.server.conf.store.TablePropKey; import org.apache.accumulo.server.init.FileSystemInitializer; import org.apache.accumulo.server.init.InitialConfiguration; +import org.apache.accumulo.server.log.DfsWalProvider; +import org.apache.accumulo.server.log.WalStateManager; import org.apache.accumulo.server.util.FindCompactionTmpFiles.DeleteStats; import org.apache.accumulo.server.util.PropUtil; import org.apache.hadoop.fs.FileStatus; @@ -226,7 +230,7 @@ public static String getRowPrefix() { @VisibleForTesting static final Set UPGRADE_FAMILIES = Set.of(DataFileColumnFamily.NAME, CHOPPED, - ExternalCompactionColumnFamily.NAME, ScanFileColumnFamily.NAME); + ExternalCompactionColumnFamily.NAME, ScanFileColumnFamily.NAME, LogColumnFamily.NAME); @VisibleForTesting static final String ZNAMESPACE_NAME = "/name"; @@ -238,6 +242,8 @@ public static String getRowPrefix() { public void upgradeZookeeper(ServerContext context) { LOG.info("Ensuring all worker server processes are down."); validateEmptyZKWorkerServerPaths(context); + LOG.info("Upgrading WAL provider IDs in ZooKeeper markers"); + upgradeWalMarkers(context); LOG.info("Removing ZTracer node"); removeZTracersNode(context); LOG.info("Updating file references in root tablet"); @@ -430,6 +436,10 @@ private void removeCompactColumnsFromRootTabletMetadata(ServerContext context) { zrw.overwritePersistentData(ZROOT_TABLET, rtm.toJson().getBytes(UTF_8), stat.getVersion()); LOG.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson()); } + RootTabletMetadata upgraded = + new RootTabletMetadata(new String(zrw.getData(ZROOT_TABLET), UTF_8)); + upgraded.getKeyValues() + .forEach(entry -> verifyNoLegacyWalReference(entry.getKey(), "root tablet metadata")); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); throw new IllegalStateException( @@ -862,6 +872,8 @@ void processReferences(MutationWriter batchWriter, Iterable tableName, key.getRow()); update.at().family(ExternalCompactionColumnFamily.NAME) .qualifier(key.getColumnQualifier()).delete(); + } else if (family.equals(LogColumnFamily.NAME)) { + upgradeWalReference(key, value, update); } else { throw new IllegalStateException("Processing: " + tableName + " Received unexpected column family processing references: " + family); @@ -879,6 +891,81 @@ void processReferences(MutationWriter batchWriter, Iterable } } + @VisibleForTesting + static void upgradeWalReference(Key key, Value value, Mutation mutation) { + String qualifier = key.getColumnQualifier().toString(); + if (qualifier.startsWith("-/")) { + Text upgradedQualifier = new Text(DfsWalProvider.PROVIDER_ID + qualifier.substring(1)); + mutation.at().family(LogColumnFamily.NAME).qualifier(upgradedQualifier).put(value); + mutation.at().family(LogColumnFamily.NAME).qualifier(key.getColumnQualifier()).delete(); + } + } + + private static void verifyNoLegacyWalReference(Key key, String source) { + if (key.getColumnFamily().equals(LogColumnFamily.NAME) + && key.getColumnQualifier().toString().startsWith("-/")) { + throw new IllegalStateException( + "Legacy WAL reference remains after upgrade in " + source + ": " + key); + } + } + + static String upgradeWalMarker(String markerData) { + String[] fields = markerData.split(",", 3); + String upgraded; + if (fields.length == 2) { + upgraded = fields[0] + "," + DfsWalProvider.PROVIDER_ID + "," + fields[1]; + } else if (fields.length == 3) { + upgraded = markerData; + } else { + throw new IllegalStateException("Malformed WAL marker: " + markerData); + } + + String[] upgradedFields = upgraded.split(",", 3); + try { + WalStateManager.WalState.valueOf(upgradedFields[0]); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Malformed WAL marker: " + markerData, e); + } + if (!upgradedFields[1].matches(WalProvider.PROVIDER_ID_REGEX) || upgradedFields[2].isBlank()) { + throw new IllegalStateException("Malformed WAL marker: " + markerData); + } + return upgraded; + } + + @VisibleForTesting + void upgradeWalMarkers(ServerContext context) { + var zrw = context.getZooSession().asReaderWriter(); + try { + String walRoot = WalStateManager.ZWALS; + if (!zrw.exists(walRoot)) { + return; + } + for (String server : zrw.getChildren(walRoot)) { + String serverPath = walRoot + "/" + server; + for (String marker : zrw.getChildren(serverPath)) { + String markerPath = serverPath + "/" + marker; + byte[] markerData = zrw.getData(markerPath); + String data = new String(markerData, UTF_8); + String upgraded = upgradeWalMarker(data); + if (!upgraded.equals(data)) { + zrw.putPersistentData(markerPath, upgraded.getBytes(UTF_8), + ZooUtil.NodeExistsPolicy.OVERWRITE); + String persisted = new String(zrw.getData(markerPath), UTF_8); + if (!persisted.equals(upgraded)) { + throw new IllegalStateException("Failed to upgrade WAL marker " + markerPath + + ": expected " + upgraded + " but found " + persisted); + } + } + } + } + } catch (KeeperException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new IllegalStateException("Error upgrading WAL provider IDs in ZooKeeper", e); + } + } + @VisibleForTesting void updateRootTabletFileReferences(ServerContext context) { try { @@ -957,6 +1044,19 @@ private void upgradeTabletsMetadata(@NonNull ServerContext context, String metaN LOG.warn("Constraint violations: {}", mex.getConstraintViolationSummaries()); throw new IllegalStateException("Failed to process table: " + metaName, mex); } + verifyNoLegacyWalReferences(context, metaName); + } + + private void verifyNoLegacyWalReferences(ServerContext context, String metaName) { + try (Scanner scanner = context.createScanner(metaName, Authorizations.EMPTY)) { + scanner.fetchColumnFamily(LogColumnFamily.NAME); + scanner.setRange(MetadataSchema.TabletsSection.getRange()); + for (Entry entry : scanner) { + verifyNoLegacyWalReference(entry.getKey(), metaName); + } + } catch (TableNotFoundException e) { + throw new IllegalStateException("Failed to verify WAL references in " + metaName, e); + } } @VisibleForTesting diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java index 002ffbeea80..0331873520e 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java @@ -92,6 +92,7 @@ import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.gc.AllVolumesDirectory; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.metadata.ConditionalTabletMutatorImpl; import org.apache.accumulo.server.metadata.ConditionalTabletsMutatorImpl; import org.apache.hadoop.io.Text; @@ -330,7 +331,8 @@ public void testUnexpectedColumns() { testUnexpectedColumn(tmb -> tmb.putOperation(otherOpid), "had unexpected opid", otherOpid.toString()); - var walog = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + var walog = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); testUnexpectedColumn(tmb -> tmb.putWal(walog), "has unexpected walogs 1"); FateId ucfid1 = otherFateId; diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java index 13ad3bde70e..b257c3be093 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java @@ -72,6 +72,7 @@ import org.apache.accumulo.manager.split.FileRangeCache; import org.apache.accumulo.manager.tableOps.FateEnv; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.metadata.ConditionalTabletMutatorImpl; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -466,7 +467,8 @@ public void testErrors() throws Exception { assertTrue(e.getMessage().contains(opid2.toString())); // Test splitting a tablet with walogs - var walog = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + var walog = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); var tablet4 = TabletMetadata.builder(origExtent).putOperation(opid).putWal(walog) .build(ColumnType.LOCATION); e = assertThrows(IllegalStateException.class, () -> testError(origExtent, tablet4, fateId)); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java index f1f1541eb89..c1f1a773372 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java @@ -72,6 +72,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LastLocationColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily; import org.apache.accumulo.core.metadata.schema.RootTabletMetadata; import org.apache.accumulo.core.zookeeper.ZooSession; @@ -82,6 +83,8 @@ import org.apache.accumulo.server.conf.store.SystemPropKey; import org.apache.accumulo.server.conf.store.TablePropKey; import org.apache.accumulo.server.conf.store.impl.ZooPropStore; +import org.apache.accumulo.server.log.DfsWalProvider; +import org.apache.accumulo.server.log.WalStateManager; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.zookeeper.KeeperException; @@ -189,6 +192,80 @@ public void processReferencesTest() { } + @Test + void upgradeWalReferenceTest() { + String path = "hdfs://nn/accumulo/wal/host+1234/11111111-1111-1111-1111-111111111111"; + Key key = Key.builder().row("123").family(LogColumnFamily.NAME).qualifier("-/" + path).build(); + Value value = new Value(); + ArrayList mutations = new ArrayList<>(); + new Upgrader11to12().processReferences(mutations::add, List.of(Map.entry(key, value)), + "accumulo.metadata"); + + assertTrue(UPGRADE_FAMILIES.contains(LogColumnFamily.NAME)); + assertEquals(1, mutations.size()); + Mutation mutation = mutations.get(0); + var updates = mutation.getUpdates(); + assertEquals(2, updates.size()); + var added = updates.stream().filter(update -> !update.isDeleted()).findFirst().orElseThrow(); + var deleted = updates.stream().filter(ColumnUpdate::isDeleted).findFirst().orElseThrow(); + assertEquals(DfsWalProvider.PROVIDER_ID + "/" + path, + new Text(added.getColumnQualifier()).toString()); + assertEquals("-/" + path, new Text(deleted.getColumnQualifier()).toString()); + + Key upgradedKey = Key.builder().row("123").family(LogColumnFamily.NAME) + .qualifier(DfsWalProvider.PROVIDER_ID + "/" + path).build(); + ArrayList retryMutations = new ArrayList<>(); + new Upgrader11to12().processReferences(retryMutations::add, + List.of(Map.entry(upgradedKey, value)), "accumulo.metadata"); + assertTrue(retryMutations.isEmpty()); + } + + @Test + void upgradeWalMarkerIsIdempotentAndRejectsInvalidProviderIds() { + String oldData = "CLOSED,hdfs://nn/accumulo/wal/host+1234/11111111-1111-1111-1111-111111111111"; + String upgraded = "CLOSED," + DfsWalProvider.PROVIDER_ID + + ",hdfs://nn/accumulo/wal/host+1234/11111111-1111-1111-1111-111111111111"; + assertEquals(upgraded, Upgrader11to12.upgradeWalMarker(oldData)); + assertEquals(upgraded, Upgrader11to12.upgradeWalMarker(upgraded)); + assertThrows(IllegalStateException.class, + () -> Upgrader11to12.upgradeWalMarker("CLOSED,provider-id,path")); + assertThrows(IllegalStateException.class, + () -> Upgrader11to12.upgradeWalMarker("CLOSED,-,path")); + } + + @Test + void upgradeWalMarkersTest() throws Exception { + ServerContext context = createMock(ServerContext.class); + ZooSession zk = createMock(ZooSession.class); + ZooReaderWriter zrw = createStrictMock(ZooReaderWriter.class); + String server = "localhost:9997[1]"; + String uuid = "11111111-1111-1111-1111-111111111111"; + String markerPath = WalStateManager.ZWALS + "/" + server + "/" + uuid; + String oldData = "CLOSED,hdfs://nn/accumulo/wal/host+1234/" + uuid; + String newData = "OPEN," + DfsWalProvider.PROVIDER_ID + + ",hdfs://nn/accumulo/wal/host+1234/22222222-2222-2222-2222-222222222222"; + + expect(context.getZooSession()).andReturn(zk); + expect(zk.asReaderWriter()).andReturn(zrw); + expect(zrw.exists(WalStateManager.ZWALS)).andReturn(true); + expect(zrw.getChildren(WalStateManager.ZWALS)).andReturn(List.of(server)); + expect(zrw.getChildren(WalStateManager.ZWALS + "/" + server)).andReturn(List.of(uuid, "222")); + expect(zrw.getData(markerPath)).andReturn(oldData.getBytes(UTF_8)); + expect(zrw.putPersistentData(markerPath, + ("CLOSED," + DfsWalProvider.PROVIDER_ID + "," + oldData.substring("CLOSED,".length())) + .getBytes(UTF_8), + ZooUtil.NodeExistsPolicy.OVERWRITE)).andReturn(true); + expect(zrw.getData(markerPath)).andReturn( + ("CLOSED," + DfsWalProvider.PROVIDER_ID + "," + oldData.substring("CLOSED,".length())) + .getBytes(UTF_8)); + expect(zrw.getData(WalStateManager.ZWALS + "/" + server + "/222")) + .andReturn(newData.getBytes(UTF_8)); + + replay(context, zk, zrw); + new Upgrader11to12().upgradeWalMarkers(context); + verify(context, zk, zrw); + } + @Test public void skipConvertedFileTest() { // create sample data "served" by the mocked scanner diff --git a/server/manager/src/test/resources/accumulo.properties b/server/manager/src/test/resources/accumulo.properties index 7a9d5c90663..1669b68ee68 100644 --- a/server/manager/src/test/resources/accumulo.properties +++ b/server/manager/src/test/resources/accumulo.properties @@ -18,3 +18,5 @@ # # File needs to exist for unit tests +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java index 0d3363c302c..0af8a0e1c15 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java @@ -215,6 +215,7 @@ private TabletMetadataLoader(Ample ample) { public ScanServer(ServerOpts opts, String[] args) { super(ServerId.Type.SCAN_SERVER, opts, ServerContext::new, args); context = super.getContext(); + context.getWalProviderRegistry(); LOG.info("Version " + Constants.VERSION); LOG.info("Instance " + getContext().getInstanceID()); this.sessionManager = new SessionManager(context); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index 1402c22865a..4a45714edf5 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@ -227,6 +227,7 @@ protected TabletServer(ServerOpts opts, String[] args) { super(ServerId.Type.TABLET_SERVER, opts, serverContextFactory, args); context = super.getContext(); + context.getWalProviderRegistry(); final AccumuloConfiguration aconf = getConfiguration(); log.info("Version " + Constants.VERSION); log.info("Instance " + getInstanceID()); @@ -1030,7 +1031,7 @@ private void markUnusedWALs() { TServerInstance session = this.getTabletSession(); for (DfsLogger candidate : eligible) { log.info("Marking " + candidate.getPath() + " as unreferenced"); - walMarker.walUnreferenced(session, candidate.getPath()); + walMarker.walUnreferenced(session, candidate.getLogEntry()); } synchronized (closedLogs) { closedLogs.removeAll(eligible); @@ -1042,7 +1043,7 @@ private void markUnusedWALs() { public void addNewLogMarker(DfsLogger copy) throws WalMarkerException { log.info("Writing log marker for " + copy.getPath()); - walMarker.addNewWalMarker(getTabletSession(), copy.getPath()); + walMarker.addNewWalMarker(getTabletSession(), copy.getLogEntry()); } public void walogClosed(DfsLogger currentLog) throws WalMarkerException { @@ -1054,7 +1055,7 @@ public void walogClosed(DfsLogger currentLog) throws WalMarkerException { clSize = closedLogs.size(); } log.info("Marking {} as closed. Total closed logs: {} ", currentLog.getPath(), clSize); - walMarker.closeWal(getTabletSession(), currentLog.getPath()); + walMarker.closeWal(getTabletSession(), currentLog.getLogEntry()); // whenever a new log is added to the set of closed logs, go through all of the tablets and // see if any need to minor compact @@ -1072,7 +1073,7 @@ public void walogClosed(DfsLogger currentLog) throws WalMarkerException { } } else { log.info("Marking {} as unreferenced (skipping closed writes == 0)", currentLog.getPath()); - walMarker.walUnreferenced(getTabletSession(), currentLog.getPath()); + walMarker.walUnreferenced(getTabletSession(), currentLog.getLogEntry()); } } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java index d021ff111bf..f0cd572378f 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java @@ -32,6 +32,7 @@ import java.io.DataOutputStream; import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.channels.ClosedChannelException; import java.time.Duration; @@ -46,7 +47,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicLong; -import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Durability; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; @@ -59,23 +59,17 @@ import org.apache.accumulo.core.spi.crypto.CryptoService; import org.apache.accumulo.core.spi.crypto.FileDecrypter; import org.apache.accumulo.core.spi.crypto.FileEncrypter; -import org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment; +import org.apache.accumulo.core.spi.wal.WalProvider; import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.core.util.Timer; import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.server.ServerContext; -import org.apache.accumulo.server.fs.VolumeChooserEnvironmentImpl; -import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.tserver.TabletMutations; import org.apache.accumulo.tserver.logger.LogFileKey; import org.apache.accumulo.tserver.logger.LogFileValue; import org.apache.accumulo.tserver.tablet.CommitSession; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSOutputStream; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,7 +91,6 @@ public final class DfsLogger implements Comparable { public static final String LOG_FILE_HEADER_V4 = "--- Log File Header (v4) ---"; private static final Logger log = LoggerFactory.getLogger(DfsLogger.class); - private static final DatanodeInfo[] EMPTY_PIPELINE = new DatanodeInfo[0]; public static class LogClosedException extends IOException { private static final long serialVersionUID = 1L; @@ -182,10 +175,10 @@ public void run() { try { if (shouldHSync.isPresent()) { if (shouldHSync.orElseThrow()) { - logFile.hsync(); + logFile.sync(Durability.SYNC); syncCounter.incrementAndGet(); } else { - logFile.hflush(); + logFile.sync(Durability.FLUSH); flushCounter.incrementAndGet(); } } @@ -198,7 +191,7 @@ public void run() { if (expectedReplication > 0) { int current = expectedReplication; try { - current = ((DFSOutputStream) logFile.getWrappedStream()).getCurrentBlockReplication(); + current = logFile.getCurrentReplication(); } catch (IOException e) { fail(work, e, "getting replication level"); } @@ -210,10 +203,9 @@ public void run() { } } } - if (expectedReplication == 0 && logFile.getWrappedStream() instanceof DFSOutputStream) { + if (expectedReplication == 0) { try { - expectedReplication = - ((DFSOutputStream) logFile.getWrappedStream()).getCurrentBlockReplication(); + expectedReplication = logFile.getCurrentReplication(); } catch (IOException e) { fail(work, e, "getting replication level"); } @@ -303,7 +295,7 @@ public int hashCode() { return logEntry.hashCode(); } - private FSDataOutputStream logFile; + private WalProvider.Writer logFile; private DataOutputStream encryptingLogFile = null; private final LogEntry logEntry; private Thread syncThread; @@ -313,24 +305,30 @@ public int hashCode() { /** * Create a new DfsLogger with the provided characteristics. */ - public static DfsLogger createNew(ServerContext context, AtomicLong syncCounter, - AtomicLong flushCounter, String address) throws IOException { - - String filename = UUID.randomUUID().toString(); - String addressForFilename = address.replace(':', '+'); - - var chooserEnv = - new VolumeChooserEnvironmentImpl(VolumeChooserEnvironment.Scope.LOGGER, context); - String logPath = - context.getVolumeManager().choose(chooserEnv, context.getBaseUris()) + Path.SEPARATOR - + Constants.WAL_DIR + Path.SEPARATOR + addressForFilename + Path.SEPARATOR + filename; - - LogEntry log = LogEntry.fromPath(logPath); - DfsLogger dfsLogger = new DfsLogger(log); - long slowFlushMillis = - context.getConfiguration().getTimeInMillis(Property.TSERV_SLOW_FLUSH_MILLIS); - dfsLogger.open(context, logPath, filename, address, syncCounter, flushCounter, slowFlushMillis); - return dfsLogger; + public static DfsLogger createNew(ServerContext context, WalProvider provider, + AtomicLong syncCounter, AtomicLong flushCounter, String address) throws IOException { + + WalProvider.Writer writer = provider.create(address, UUID.randomUUID()); + DfsLogger dfsLogger = null; + try { + LogEntry log = LogEntry.fromPath(provider.getId(), writer.getReference()); + dfsLogger = new DfsLogger(log); + long slowFlushMillis = + context.getConfiguration().getTimeInMillis(Property.TSERV_SLOW_FLUSH_MILLIS); + dfsLogger.open(context, writer, address, syncCounter, flushCounter, slowFlushMillis); + return dfsLogger; + } catch (IOException | RuntimeException e) { + try { + if (dfsLogger != null && dfsLogger.syncThread != null) { + dfsLogger.close(); + } else { + writer.close(); + } + } catch (IOException closeException) { + e.addSuppressed(closeException); + } + throw e; + } } /** @@ -353,33 +351,35 @@ private DfsLogger(LogEntry logEntry) { * @throws LogHeaderIncompleteException if the header cannot be fully read (can happen if the * tserver died before finishing) */ - public static DataInputStream getDecryptingStream(FSDataInputStream input, - CryptoService cryptoService) throws LogHeaderIncompleteException, IOException { + public static DataInputStream getDecryptingStream(InputStream input, CryptoService cryptoService) + throws LogHeaderIncompleteException, IOException { DataInputStream decryptingInput; + DataInputStream dataInput = + input instanceof DataInputStream dis ? dis : new DataInputStream(input); byte[] magic4 = DfsLogger.LOG_FILE_HEADER_V4.getBytes(UTF_8); byte[] magic3 = DfsLogger.LOG_FILE_HEADER_V3.getBytes(UTF_8); byte[] magicBuffer = new byte[magic4.length]; try { - input.readFully(magicBuffer); + dataInput.readFully(magicBuffer); if (Arrays.equals(magicBuffer, magic4)) { FileDecrypter decrypter = - CryptoUtils.getFileDecrypter(cryptoService, Scope.WAL, null, input); + CryptoUtils.getFileDecrypter(cryptoService, Scope.WAL, null, dataInput); log.debug("Using {} for decrypting WAL", cryptoService.getClass().getSimpleName()); - var stream = decrypter.decryptStream(input); + var stream = decrypter.decryptStream(dataInput); decryptingInput = stream instanceof DataInputStream ? (DataInputStream) stream : new DataInputStream(stream); } else if (Arrays.equals(magicBuffer, magic3)) { // Read logs files from Accumulo 1.9 and throw an error if they are encrypted - String cryptoModuleClassname = input.readUTF(); + String cryptoModuleClassname = dataInput.readUTF(); if (!cryptoModuleClassname.equals("NullCryptoModule")) { throw new IllegalArgumentException( "Old encryption modules not supported at this time. Unsupported module : " + cryptoModuleClassname); } - decryptingInput = input; + decryptingInput = dataInput; } else { throw new IllegalArgumentException( "Unsupported write ahead log version " + new String(magicBuffer, UTF_8)); @@ -401,62 +401,35 @@ public static DataInputStream getDecryptingStream(FSDataInputStream input, * * @param address The address of the host using this WAL */ - private synchronized void open(ServerContext context, String logPath, String filename, - String address, AtomicLong syncCounter, AtomicLong flushCounter, long slowFlushMillis) - throws IOException { + private synchronized void open(ServerContext context, WalProvider.Writer writer, String address, + AtomicLong syncCounter, AtomicLong flushCounter, long slowFlushMillis) throws IOException { log.debug("Address is {}", address); log.debug("DfsLogger.open() begin"); - VolumeManager fs = context.getVolumeManager(); - LoggerOperation op; - var serverConf = context.getConfiguration(); try { - Path logfilePath = new Path(logPath); - short replication = (short) serverConf.getCount(Property.TSERV_WAL_REPLICATION); - if (replication == 0) { - replication = fs.getDefaultReplication(logfilePath); - } - long blockSize = getWalBlockSize(serverConf); - if (serverConf.getBoolean(Property.TSERV_WAL_SYNC)) { - logFile = fs.createSyncable(logfilePath, 0, replication, blockSize); - } else { - logFile = fs.create(logfilePath, true, 0, replication, blockSize); - } - - // Tell the DataNode that the write ahead log does not need to be cached in the OS page cache - try { - logFile.setDropBehind(Boolean.TRUE); - } catch (UnsupportedOperationException e) { - log.debug("setDropBehind writes not enabled for wal file: {}", logFile); - } catch (IOException e) { - log.debug("IOException setting drop behind for file: {}, msg: {}", logFile, e.getMessage()); - } - - // check again that logfile can be sync'd - if (!fs.canSyncAndFlush(logfilePath)) { - log.warn("sync not supported for log file {}. Data loss may occur.", logPath); - } + logFile = writer; + OutputStream output = logFile.getOutputStream(); // Initialize the log file with a header and its encryption CryptoEnvironment env = new CryptoEnvironmentImpl(Scope.WAL); - CryptoService cryptoService = - context.getCryptoFactory().getService(env, serverConf.getAllCryptoProperties()); - logFile.write(LOG_FILE_HEADER_V4.getBytes(UTF_8)); + CryptoService cryptoService = context.getCryptoFactory().getService(env, + context.getConfiguration().getAllCryptoProperties()); + output.write(LOG_FILE_HEADER_V4.getBytes(UTF_8)); log.debug("Using {} for encrypting WAL {}", cryptoService.getClass().getSimpleName(), - filename); + writer.getReference()); FileEncrypter encrypter = cryptoService.getFileEncrypter(env); byte[] cryptoParams = encrypter.getDecryptionParameters(); - CryptoUtils.writeParams(cryptoParams, logFile); + CryptoUtils.writeParams(cryptoParams, new DataOutputStream(output)); /* * Always wrap the WAL in a NoFlushOutputStream to prevent extra flushing to HDFS. The method * write(LogFileKey, LogFileValue) will flush crypto data or do nothing when crypto is not * enabled. */ - OutputStream encryptedStream = encrypter.encryptStream(new NoFlushOutputStream(logFile)); + OutputStream encryptedStream = encrypter.encryptStream(new NoFlushOutputStream(output)); if (encryptedStream instanceof NoFlushOutputStream) { encryptingLogFile = (NoFlushOutputStream) encryptedStream; } else { @@ -465,8 +438,8 @@ private synchronized void open(ServerContext context, String logPath, String fil LogFileKey key = new LogFileKey(); key.setEvent(OPEN); - key.setTserverSession(filename); - key.setFilename(filename); + key.setTserverSession(logEntry.getUniqueID().toString()); + key.setFilename(logEntry.getUniqueID().toString()); op = logKeyData(key, Durability.SYNC); } catch (Exception ex) { if (logFile != null) { @@ -551,9 +524,9 @@ public synchronized long getWrites() { return writes; } - public synchronized long getBytesWritten() { + public synchronized long getBytesWritten() throws IOException { Objects.requireNonNull(logFile, "logFile is null"); - return logFile.getPos(); + return logFile.getLength(); } public LoggerOperation defineTablet(CommitSession cs) throws IOException { @@ -677,18 +650,10 @@ public int compareTo(DfsLogger o) { /** * This method gets the pipeline for the current walog. * - * @return non-null array of DatanodeInfo + * @return diagnostic storage pipeline */ - DatanodeInfo[] getPipeLine() { - if (logFile != null) { - OutputStream os = logFile.getWrappedStream(); - if (os instanceof DFSOutputStream) { - return ((DFSOutputStream) os).getPipeline(); - } - } - - // Don't have a pipeline or can't figure it out. - return EMPTY_PIPELINE; + String[] getPipeLine() { + return logFile == null ? new String[0] : logFile.getPipeline(); } } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java index 93a23b52e4d..abe80ddc95b 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java @@ -27,6 +27,7 @@ import java.io.DataInputStream; import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -51,18 +52,19 @@ import org.apache.accumulo.core.metrics.MetricsProducer; import org.apache.accumulo.core.spi.crypto.CryptoEnvironment; import org.apache.accumulo.core.spi.crypto.CryptoService; +import org.apache.accumulo.core.util.CountingInputStream; import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.server.AbstractServer; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.SortedLogState; import org.apache.accumulo.server.zookeeper.DistributedWorkQueue; import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor; import org.apache.accumulo.tserver.log.DfsLogger.LogHeaderIncompleteException; import org.apache.accumulo.tserver.logger.LogFileKey; import org.apache.accumulo.tserver.logger.LogFileValue; -import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.zookeeper.KeeperException; @@ -84,7 +86,7 @@ public class LogSorter implements MetricsProducer { class LogProcessor implements Processor { - private FSDataInputStream input; + private CountingInputStream input; private DataInputStream decryptingInput; private long bytesCopied = -1; private long sortStart = 0; @@ -99,8 +101,9 @@ public Processor newProcessor() { public void process(String child, byte[] data) { String work = new String(data, UTF_8); String[] parts = work.split("\\|"); - String src = parts[0]; - String dest = parts[1]; + String providerId = parts.length == 2 ? DfsWalProvider.PROVIDER_ID : parts[0]; + String src = parts.length == 2 ? parts[0] : parts[1]; + String dest = parts.length == 2 ? parts[1] : parts[2]; String sortId = new Path(src).getName(); log.debug("Sorting {} to {} using sortId {}", src, dest, sortId); @@ -119,7 +122,7 @@ public void process(String child, byte[] data) { String formerThreadName = Thread.currentThread().getName(); try { - sort(fs, sortId, new Path(src), dest); + sort(fs, sortId, providerId, new Path(src), dest); } catch (Exception t) { try { // parent dir may not exist @@ -145,6 +148,11 @@ public void process(String child, byte[] data) { public void sort(VolumeManager fs, String name, Path srcPath, String destPath) throws IOException { + sort(fs, name, DfsWalProvider.PROVIDER_ID, srcPath, destPath); + } + + public void sort(VolumeManager fs, String name, String providerId, Path srcPath, + String destPath) throws IOException { int part = 0; // check for finished first since another thread may have already done the sort @@ -157,16 +165,9 @@ public void sort(VolumeManager fs, String name, Path srcPath, String destPath) // the following call does not throw an exception if the file/dir does not exist fs.deleteRecursively(new Path(destPath)); - input = fs.open(srcPath); - - // Tell the DataNode that the write ahead log does not need to be cached in the OS page cache - try { - input.setDropBehind(Boolean.TRUE); - } catch (UnsupportedOperationException e) { - log.debug("setDropBehind reads not enabled for wal file: {}", input); - } catch (IOException e) { - log.debug("IOException setting drop behind for file: {}, msg: {}", input, e.getMessage()); - } + InputStream source = providerId.equals(DfsWalProvider.PROVIDER_ID) ? fs.open(srcPath) + : context.getWalProviderRegistry().get(providerId).open(srcPath.toString()); + input = new CountingInputStream(source); try { decryptingInput = DfsLogger.getDecryptingStream(input, cryptoService); @@ -187,8 +188,8 @@ public void sort(VolumeManager fs, String name, Path srcPath, String destPath) while (true) { final ArrayList> buffer = new ArrayList<>(512); try { - long start = input.getPos(); - while (input.getPos() - start < bufferSize) { + long start = input.getCount(); + while (input.getCount() - start < bufferSize) { LogFileKey key = new LogFileKey(); LogFileValue value = new LogFileValue(); key.readFields(decryptingInput); @@ -211,7 +212,7 @@ synchronized void close() throws IOException { // If we receive an empty or malformed-header WAL, we won't // have input streams that need closing. Avoid the NPE. if (input != null) { - bytesCopied = input.getPos(); + bytesCopied = input.getCount(); input.close(); if (decryptingInput != null) { decryptingInput.close(); @@ -231,7 +232,7 @@ public synchronized long getSortTime() { } synchronized long getBytesCopied() throws IOException { - return input == null ? bytesCopied : input.getPos(); + return input == null ? bytesCopied : input.getCount(); } } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java index 6843538510b..9f511a383e0 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java @@ -275,7 +275,8 @@ private synchronized void startLogMaker() { final DfsLogger alog; try { - alog = DfsLogger.createNew(tserver.getContext(), syncCounter, flushCounter, + alog = DfsLogger.createNew(tserver.getContext(), + tserver.getContext().getWalProviderRegistry().getCurrent(), syncCounter, flushCounter, tserver.getAdvertiseAddress().toString()); } catch (IOException | RuntimeException e) { log.error("Failed to open WAL", e); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java index c41455850ff..1056c586ad3 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/logger/LogReader.java @@ -45,6 +45,7 @@ import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.util.ServerKeywordExecutable; import org.apache.accumulo.start.spi.CommandGroup; import org.apache.accumulo.start.spi.CommandGroups; @@ -175,7 +176,8 @@ public void execute(JCommander cl, ReaderOpts opts) throws Exception { } else { // read the log entries in a sorted RFile. This has to be a directory that contains the // finished file. - var rsl = ResolvedSortedLog.resolve(LogEntry.fromPath(path.toString()), fs); + var rsl = ResolvedSortedLog + .resolve(LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, path.toString()), fs); try (var rli = new RecoveryLogsIterator(context, Collections.singletonList(rsl), null, null, false)) { while (rli.hasNext()) { diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/WalRemovalOrderTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/WalRemovalOrderTest.java index 2e682b4231a..80c35416552 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/WalRemovalOrderTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/WalRemovalOrderTest.java @@ -26,6 +26,7 @@ import java.util.Set; import org.apache.accumulo.core.tabletserver.log.LogEntry; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.tserver.log.DfsLogger; import org.junit.jupiter.api.Test; @@ -34,7 +35,8 @@ public class WalRemovalOrderTest { private DfsLogger mockLogger(String filename) { - var mockLogEntry = LogEntry.fromPath(filename + "+1234/11111111-1111-1111-1111-111111111111"); + var mockLogEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, + filename + "+1234/11111111-1111-1111-1111-111111111111"); return DfsLogger.fromLogEntry(mockLogEntry); } diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsIteratorTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsIteratorTest.java index 76516fbed84..618256dd723 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsIteratorTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsIteratorTest.java @@ -48,6 +48,7 @@ import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.fs.VolumeManagerImpl; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.SortedLogState; import org.apache.accumulo.tserver.TabletServer; import org.apache.accumulo.tserver.WithTestNames; @@ -252,7 +253,8 @@ private void createRecoveryDir(Map logs, ArrayList recover(Map logs, Set files, K logSorter.writeBuffer(destPath, buffer, parts); ns.create(SortedLogState.getFinishedMarkerPath(destPath)).close(); - dirs.add(ResolvedSortedLog.resolve(LogEntry.fromPath(origPath), fs)); + dirs.add( + ResolvedSortedLog.resolve(LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, origPath), fs)); } // Recover SortedLogRecovery recovery = new SortedLogRecovery(context, fileLenCache, cacheProvider); diff --git a/server/tserver/src/test/resources/accumulo.properties b/server/tserver/src/test/resources/accumulo.properties index b63ed5c0617..2a7b4af5260 100644 --- a/server/tserver/src/test/resources/accumulo.properties +++ b/server/tserver/src/test/resources/accumulo.properties @@ -21,3 +21,5 @@ instance.dfs.dir=${project.build.directory}/instanceTest instance.secret=TEST_SYSTEM_SECRET +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider diff --git a/shell/pom.xml b/shell/pom.xml index b413fdcb4c3..08294cf7a4d 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -91,6 +91,11 @@ hadoop-client-runtime runtime + + org.apache.accumulo + accumulo-server-base + test + org.apache.logging.log4j log4j-slf4j2-impl diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java index a12c23e3c5a..48f4b3c93f2 100644 --- a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java +++ b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java @@ -48,6 +48,7 @@ import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.tabletserver.log.LogEntry; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.shell.Shell; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -191,8 +192,10 @@ public void mockTest() throws Exception { KeyExtent extent = new KeyExtent(tableId, new Text("d"), null); - LogEntry le1 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); - LogEntry le2 = LogEntry.fromPath("localhost+8020/" + UUID.randomUUID()); + LogEntry le1 = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); + LogEntry le2 = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); TabletMetadata tm1 = TabletMetadata.builder(extent).putTabletAvailability(TabletAvailability.ONDEMAND) diff --git a/test/src/main/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java b/test/src/main/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java index e33abf148bf..c4f01f77bef 100644 --- a/test/src/main/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java +++ b/test/src/main/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java @@ -41,6 +41,7 @@ import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.accumulo.tserver.log.DfsLogger; import org.apache.hadoop.conf.Configuration; @@ -136,7 +137,8 @@ public void testEmptyWalRecoveryCompletes() throws Exception { TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName)); assertNotNull(tableId, "Table ID was null"); - LogEntry logEntry = LogEntry.fromPath(emptyWalog.toUri().toString()); + LogEntry logEntry = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, emptyWalog.toUri().toString()); log.info("Taking {} offline", tableName); client.tableOperations().offline(tableName, true); @@ -196,7 +198,8 @@ public void testPartialHeaderWalRecoveryCompletes() throws Exception { TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName)); assertNotNull(tableId, "Table ID was null"); - LogEntry logEntry = LogEntry.fromPath(partialHeaderWalog.toUri().toString()); + LogEntry logEntry = + LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, partialHeaderWalog.toUri().toString()); log.info("Taking {} offline", tableName); client.tableOperations().offline(tableName, true); diff --git a/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java b/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java index 6f39b63be11..d238d3dab7d 100644 --- a/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java +++ b/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java @@ -90,6 +90,7 @@ import org.apache.accumulo.manager.tableOps.split.PreSplit; import org.apache.accumulo.manager.tableOps.split.SplitInfo; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.test.LargeSplitRowIT; import org.apache.accumulo.test.ample.metadata.TestAmple; import org.apache.accumulo.test.ample.metadata.TestAmple.TestServerAmpleImpl; @@ -161,7 +162,8 @@ public void testNoWalsMergeRepos(MergeInfo.Operation operation) throws Exception // Write a WAL to the test metadata and then re-run the repo to check for an error try (TabletsMutator tm = testAmple.mutateTablets()) { var walFilePath = Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - tm.mutateTablet(extent).putWal(LogEntry.fromPath(walFilePath)).mutate(); + tm.mutateTablet(extent).putWal(LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath)) + .mutate(); } // Should not be ready due to the presence of a WAL diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java index 713eabdf82b..8ae7c688ee2 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java @@ -105,6 +105,7 @@ import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.server.ServerContext; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.metadata.AsyncConditionalTabletsMutatorImpl; import org.apache.accumulo.test.harness.SharedMiniClusterBase; import org.apache.accumulo.test.util.Wait; @@ -473,7 +474,7 @@ public void testWALs() { // Test adding a WAL to a tablet and verifying its presence String walFilePath = java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - final LogEntry originalLogEntry = LogEntry.fromPath(walFilePath); + final LogEntry originalLogEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath); // create a tablet metadata with no write ahead logs var tmEmptySet = TabletMetadata.builder(e1).build(LOGS); // tablet should not have any logs to start with so requireSame with the empty logs should pass @@ -491,7 +492,7 @@ public void testWALs() { // Test adding another WAL and verifying the update String walFilePath2 = java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - LogEntry newLogEntry = LogEntry.fromPath(walFilePath2); + LogEntry newLogEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath2); try (var ctmi = context.getAmple().conditionallyMutateTablets()) { ctmi.mutateTablet(e1).requireAbsentOperation().putWal(newLogEntry).submit(tm -> false); assertEquals(Status.ACCEPTED, ctmi.process().get(e1).getStatus()); @@ -504,7 +505,7 @@ public void testWALs() { String walFilePath3 = java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - LogEntry otherLogEntry = LogEntry.fromPath(walFilePath3); + LogEntry otherLogEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath3); // create a powerset to ensure all possible subsets fail when using requireSame except the // expected current state @@ -1236,7 +1237,7 @@ public void multipleFilters() { String server = "server1+8555"; String walFilePath = java.nio.file.Path.of(server, UUID.randomUUID().toString()).toString(); - LogEntry wal = LogEntry.fromPath(walFilePath); + LogEntry wal = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath); // add wal compact and flush ID to these tablets final Set tabletsWithWalCompactFlush = Set.of(e1, e2, e3); @@ -1340,7 +1341,7 @@ public void walFilter() { // add a wal to e2 var walFilePath = java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - var wal = LogEntry.fromPath(walFilePath); + var wal = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath); try (var ctmi = context.getAmple().conditionallyMutateTablets()) { ctmi.mutateTablet(e2).requireAbsentOperation().putWal(wal).submit(tabletMetadata -> false); assertEquals(Status.ACCEPTED, ctmi.process().get(e2).getStatus()); @@ -1351,7 +1352,7 @@ public void walFilter() { // add wal to tablet e4 walFilePath = java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - wal = LogEntry.fromPath(walFilePath); + wal = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath); try (var ctmi = context.getAmple().conditionallyMutateTablets()) { ctmi.mutateTablet(e4).requireAbsentOperation().putWal(wal).submit(tabletMetadata -> false); assertEquals(Status.ACCEPTED, ctmi.process().get(e4).getStatus()); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/InMemoryWalProviderIT.java b/test/src/main/java/org/apache/accumulo/test/functional/InMemoryWalProviderIT.java new file mode 100644 index 00000000000..35ac1688a04 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/functional/InMemoryWalProviderIT.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.functional; + +import static org.apache.accumulo.minicluster.ServerType.GARBAGE_COLLECTOR; +import static org.apache.accumulo.minicluster.ServerType.TABLET_SERVER; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.metadata.SystemTables; +import org.apache.accumulo.core.spi.wal.WalProvider; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; +import org.apache.accumulo.server.log.DfsWalProvider; +import org.apache.accumulo.server.log.WalStateManager; +import org.apache.accumulo.server.log.WalStateManager.WalStatePath; +import org.apache.accumulo.test.TestIngest; +import org.apache.accumulo.test.VerifyIngest; +import org.apache.accumulo.test.VerifyIngest.VerifyParams; +import org.apache.accumulo.test.util.Wait; +import org.apache.accumulo.test.wal.InMemoryWalProvider; +import org.apache.accumulo.test.wal.InMemoryWalStore; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +public class InMemoryWalProviderIT extends ConfigurableMacBase { + + private InMemoryWalStore walStore; + + @Override + protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { + try { + if (walStore == null) { + walStore = InMemoryWalStore.start(); + } + } catch (IOException e) { + throw new IllegalStateException("Unable to start in-memory WAL store", e); + } + + var options = walStore.getProviderOptions(); + String providerClass = InMemoryWalProvider.class.getName(); + String providerConfig = "{\"" + DfsWalProvider.class.getName() + "\":{},\"" + providerClass + + "\":{\"host\":\"" + options.get(InMemoryWalProvider.HOST_OPTION) + "\",\"port\":\"" + + options.get(InMemoryWalProvider.PORT_OPTION) + "\"}}"; + cfg.setProperty(Property.INSTANCE_WAL_PROVIDER, providerClass); + cfg.setProperty(Property.INSTANCE_WAL_PROVIDERS, providerConfig); + cfg.setProperty(Property.TSERV_WAL_MAX_SIZE, "1M"); + cfg.setProperty(Property.GC_CYCLE_START, "0s"); + cfg.setProperty(Property.GC_CYCLE_DELAY, "1s"); + cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s"); + cfg.setProperty(Property.MANAGER_RECOVERY_DELAY, "1s"); + cfg.getClusterServerConfiguration().setNumDefaultTabletServers(1); + hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName()); + } + + @AfterEach + @Override + public void tearDown() { + try { + super.tearDown(); + } finally { + if (walStore != null) { + walStore.close(); + walStore = null; + } + } + } + + @Test + public void recoversAndGarbageCollectsWalThroughProvider() throws Exception { + var control = getCluster().getClusterControl(); + control.stop(GARBAGE_COLLECTOR); + try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) { + String tableName = getUniqueNames(1)[0]; + client.tableOperations().create(tableName); + + VerifyParams params = new VerifyParams(getClientProperties(), tableName); + params.rows = 2_000; + params.dataSize = 1_024; + TestIngest.ingest(client, params); + VerifyIngest.verifyIngest(client, params); + + Set walStates = new WalStateManager(getServerContext()).getAllState(); + var provider = + getServerContext().getWalProviderRegistry().get(InMemoryWalProvider.PROVIDER_ID); + assertTrue(getServerContext().getWalProviderRegistry().getIds() + .containsAll(Set.of(DfsWalProvider.PROVIDER_ID, InMemoryWalProvider.PROVIDER_ID))); + Set walReferences = walStates.stream() + .filter(state -> state.providerId().equals(InMemoryWalProvider.PROVIDER_ID)) + .map(state -> state.path().toString()).collect(Collectors.toSet()); + assertFalse(walReferences.isEmpty(), "SPI-created WALs should be present"); + for (String reference : walReferences) { + assertTrue(provider.exists(reference), "SPI-created WAL should exist: " + reference); + } + + control.stopAllServers(TABLET_SERVER); + control.startAllServers(TABLET_SERVER); + VerifyIngest.verifyIngest(client, params); + client.tableOperations().flush(tableName, null, null, true); + for (String systemTable : new String[] {SystemTables.METADATA.tableName(), + SystemTables.ROOT.tableName(), SystemTables.FATE.tableName()}) { + client.tableOperations().flush(systemTable, null, null, true); + } + + control.start(GARBAGE_COLLECTOR); + Wait.waitFor(() -> anyWalDeleted(provider, walReferences), + "Garbage collector did not delete recovered WALs through their provider"); + } + } + + private static boolean anyWalDeleted(WalProvider provider, Set references) + throws IOException { + for (String reference : references) { + if (!provider.exists(reference)) { + return true; + } + } + return false; + } +} diff --git a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java index 8936f6e80bd..5516db2e9b1 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java @@ -99,6 +99,7 @@ import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.time.SteadyTime; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; +import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.manager.LiveTServerSet; import org.apache.accumulo.server.manager.state.TabletManagementIterator; import org.apache.accumulo.server.manager.state.TabletManagementParameters; @@ -672,7 +673,7 @@ private void createLogEntry(AccumuloClient client, String table, String tableNam KeyExtent extent = new KeyExtent(tableIdToModify, new Text("some split"), null); Mutation m = new Mutation(extent.toMetaRow()); String fileName = "file:/accumulo/wal/localhost+9997/" + UUID.randomUUID().toString(); - LogEntry logEntry = LogEntry.fromPath(fileName); + LogEntry logEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, fileName); logEntry.addToMutation(m); try (BatchWriter bw = client.createBatchWriter(table)) { bw.addMutation(m); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/WalProviderSwitchIT.java b/test/src/main/java/org/apache/accumulo/test/functional/WalProviderSwitchIT.java new file mode 100644 index 00000000000..25ba6c75755 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/functional/WalProviderSwitchIT.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.functional; + +import static org.apache.accumulo.core.lock.ServiceLockPaths.AddressSelector.all; +import static org.apache.accumulo.core.lock.ServiceLockPaths.ResourceGroupPredicate.DEFAULT_RG_ONLY; +import static org.apache.accumulo.minicluster.ServerType.GARBAGE_COLLECTOR; +import static org.apache.accumulo.minicluster.ServerType.TABLET_SERVER; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.lock.ServiceLock; +import org.apache.accumulo.core.lock.ServiceLockPaths.ServiceLockPath; +import org.apache.accumulo.core.spi.wal.WalProvider; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; +import org.apache.accumulo.miniclusterImpl.ProcessReference; +import org.apache.accumulo.server.log.DfsWalProvider; +import org.apache.accumulo.server.log.WalStateManager; +import org.apache.accumulo.server.log.WalStateManager.WalState; +import org.apache.accumulo.server.log.WalStateManager.WalStatePath; +import org.apache.accumulo.test.TestIngest; +import org.apache.accumulo.test.VerifyIngest; +import org.apache.accumulo.test.VerifyIngest.VerifyParams; +import org.apache.accumulo.test.util.Wait; +import org.apache.accumulo.test.wal.InMemoryWalProvider; +import org.apache.accumulo.test.wal.InMemoryWalStore; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +public class WalProviderSwitchIT extends ConfigurableMacBase { + + private InMemoryWalStore walStore; + + @Override + protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { + try { + if (walStore == null) { + walStore = InMemoryWalStore.start(); + } + } catch (IOException e) { + throw new IllegalStateException("Unable to start in-memory WAL store", e); + } + + var options = walStore.getProviderOptions(); + String memoryProviderClass = InMemoryWalProvider.class.getName(); + String providers = "{\"" + DfsWalProvider.class.getName() + "\":{},\"" + memoryProviderClass + + "\":{\"host\":\"" + options.get(InMemoryWalProvider.HOST_OPTION) + "\",\"port\":\"" + + options.get(InMemoryWalProvider.PORT_OPTION) + "\"}}"; + cfg.setProperty(Property.INSTANCE_WAL_PROVIDER, DfsWalProvider.class.getName()); + cfg.setProperty(Property.INSTANCE_WAL_PROVIDERS, providers); + cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s"); + cfg.setProperty(Property.MANAGER_RECOVERY_DELAY, "1s"); + cfg.setProperty(Property.GC_CYCLE_START, "0s"); + cfg.setProperty(Property.GC_CYCLE_DELAY, "1s"); + cfg.setProperty(Property.TSERV_WAL_MAX_SIZE, "2M"); + cfg.getClusterServerConfiguration().setNumDefaultTabletServers(1); + hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName()); + } + + @AfterEach + @Override + public void tearDown() { + try { + super.tearDown(); + } finally { + if (walStore != null) { + walStore.close(); + walStore = null; + } + } + } + + @Test + public void recoversDfsWalAfterSwitchingCurrentProvider() throws Exception { + var control = getCluster().getClusterControl(); + control.stop(GARBAGE_COLLECTOR); + + try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) { + String tableName = getUniqueNames(1)[0]; + client.tableOperations().create(tableName); + client.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "sync"); + + VerifyParams params = new VerifyParams(getClientProperties(), tableName); + params.rows = 2_000; + params.dataSize = 1_024; + TestIngest.ingest(client, params); + VerifyIngest.verifyIngest(client, params); + + Set initialWals = new WalStateManager(getServerContext()).getAllState(); + Set dfsReferences = + initialWals.stream().filter(wal -> wal.providerId().equals(DfsWalProvider.PROVIDER_ID)) + .map(wal -> wal.path().toString()).collect(Collectors.toSet()); + assertFalse(dfsReferences.isEmpty(), "Initial WALs should use the DFS provider"); + + Set tabletServers = + getCluster().getProcesses().get(TABLET_SERVER).stream().collect(Collectors.toSet()); + assertFalse(tabletServers.isEmpty(), "Expected a running TabletServer process"); + ServiceLockPath tabletServerLock = getServerContext().getServerPaths() + .getTabletServer(DEFAULT_RG_ONLY, all(), true).stream().findFirst().orElseThrow(); + ServiceLock.deleteLock(getServerContext().getZooSession().asReaderWriter(), tabletServerLock); + + Wait.waitFor(() -> tabletServers.stream().noneMatch(ref -> ref.getProcess().isAlive()), + "TabletServer did not exit after its ZooKeeper lock was removed"); + + // Preserve the WALs and metadata, but stop all processes before changing the instance-wide + // site property. The WAL provider map continues to include both providers. + getCluster().stop(); + setCurrentWalProvider(InMemoryWalProvider.class.getName()); + getCluster().start(); + + VerifyIngest.verifyIngest(client, params); + + VerifyParams newRows = new VerifyParams(getClientProperties(), tableName); + newRows.startRow = params.rows; + newRows.rows = 1; + newRows.dataSize = params.dataSize; + TestIngest.ingest(client, newRows); + VerifyIngest.verifyIngest(client, newRows); + + WalProvider memoryProvider = + getServerContext().getWalProviderRegistry().get(InMemoryWalProvider.PROVIDER_ID); + Set walsAfterSwitch = new WalStateManager(getServerContext()).getAllState(); + Set memoryReferences = walsAfterSwitch.stream() + .filter(wal -> wal.state() == WalState.OPEN + && wal.providerId().equals(InMemoryWalProvider.PROVIDER_ID)) + .map(wal -> wal.path().toString()).collect(Collectors.toSet()); + assertFalse(memoryReferences.isEmpty(), "New WALs should use the in-memory provider"); + for (String reference : memoryReferences) { + assertTrue(memoryProvider.exists(reference), "In-memory WAL should exist: " + reference); + } + } + } + + private void setCurrentWalProvider(String providerClass) throws IOException { + var config = getCluster().getConfig(); + config.setProperty(Property.INSTANCE_WAL_PROVIDER, providerClass); + Properties siteProperties = new Properties(); + siteProperties.putAll(config.getSiteConfig()); + try (OutputStream output = + Files.newOutputStream(config.getConfDir().toPath().resolve("accumulo.properties"))) { + siteProperties.store(output, "Updated WAL provider for integration test restart"); + } + } +} diff --git a/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java new file mode 100644 index 00000000000..eee1b1c817e --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.wal; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; + +import org.apache.accumulo.core.client.Durability; +import org.apache.accumulo.core.spi.wal.WalProvider; + +/** + * In-memory WAL provider intended for integration tests using MiniAccumuloCluster. WAL bytes are + * held by an {@link InMemoryWalStore} running in the test process and accessed over loopback TCP, + * allowing the cluster's separate server processes to share them. Start the store before starting + * the cluster and pass {@link InMemoryWalStore#getProviderOptions()} as this provider's options. + * References use a local filesystem URI with the standard WAL path layout so Accumulo can place + * sorted recovery logs in the corresponding recovery directory; the WAL bytes themselves remain in + * the in-memory store. Writes are buffered in the provider process and sent to the store on + * {@code sync} or {@code close}; therefore synced bytes remain available for recovery after a + * TabletServer process exits, while unsynced bytes may be lost with that process. + */ +public class InMemoryWalProvider implements WalProvider { + + public static final String PROVIDER_ID = "mem"; + public static final String HOST_OPTION = "host"; + public static final String PORT_OPTION = "port"; + private static final String SCHEME = "file"; + + private String host; + private int port = -1; + + @Override + public String getId() { + return PROVIDER_ID; + } + + @Override + public void init(InitParameters params) { + Map options = params.getOptions(); + host = options.get(HOST_OPTION); + if (host == null || host.isBlank()) { + throw new IllegalArgumentException( + "Missing in-memory WAL store option '" + HOST_OPTION + "'"); + } + String portOption = options.get(PORT_OPTION); + if (portOption == null) { + throw new IllegalArgumentException( + "Missing in-memory WAL store option '" + PORT_OPTION + "'"); + } + try { + port = Integer.parseInt(portOption); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid in-memory WAL store port: " + portOption, e); + } + if (port < 1 || port > 65535) { + throw new IllegalArgumentException("Invalid in-memory WAL store port: " + port); + } + } + + @Override + public Writer create(String serverAddress, UUID id) throws IOException { + requireInitialized(); + String hostForPath = Objects.requireNonNull(serverAddress).replace(':', '+'); + String reference; + try { + reference = + new URI(SCHEME, null, referenceRoot() + hostForPath + "/" + id, null, null).toString(); + } catch (URISyntaxException e) { + throw new IOException("Invalid TabletServer address for WAL reference: " + serverAddress, e); + } + validateReference(reference); + try (var socket = InMemoryWalStore.connect(host, port); + var output = new DataOutputStream(socket.getOutputStream()); + var input = new DataInputStream(socket.getInputStream())) { + InMemoryWalStore.sendRequest(output, InMemoryWalStore.CREATE, reference); + output.flush(); + InMemoryWalStore.checkResponse(input); + } + return new MemoryWriter(reference); + } + + @Override + public InputStream open(String reference) throws IOException { + requireInitialized(); + validateReference(reference); + try (var socket = InMemoryWalStore.connect(host, port); + var output = new DataOutputStream(socket.getOutputStream()); + var input = new DataInputStream(socket.getInputStream())) { + InMemoryWalStore.sendRequest(output, InMemoryWalStore.READ, reference); + output.flush(); + InMemoryWalStore.checkResponse(input); + int length = input.readInt(); + if (length < 0) { + throw new IOException("Invalid in-memory WAL length: " + length); + } + byte[] bytes = input.readNBytes(length); + if (bytes.length != length) { + throw new IOException("Incomplete WAL contents for " + reference); + } + return new ByteArrayInputStream(bytes); + } + } + + @Override + public boolean exists(String reference) throws IOException { + requireInitialized(); + validateReference(reference); + try (var socket = InMemoryWalStore.connect(host, port); + var output = new DataOutputStream(socket.getOutputStream()); + var input = new DataInputStream(socket.getInputStream())) { + InMemoryWalStore.sendRequest(output, InMemoryWalStore.EXISTS, reference); + output.flush(); + InMemoryWalStore.checkResponse(input); + return input.readBoolean(); + } + } + + @Override + public void delete(String reference) throws IOException { + requireInitialized(); + validateReference(reference); + try (var socket = InMemoryWalStore.connect(host, port); + var output = new DataOutputStream(socket.getOutputStream()); + var input = new DataInputStream(socket.getInputStream())) { + InMemoryWalStore.sendRequest(output, InMemoryWalStore.DELETE, reference); + output.flush(); + InMemoryWalStore.checkResponse(input); + } + } + + private void requireInitialized() throws IOException { + if (host == null || port < 1) { + throw new IOException("InMemoryWalProvider has not been initialized"); + } + } + + private void validateReference(String reference) throws IOException { + try { + URI uri = new URI(reference); + if (!SCHEME.equals(uri.getScheme()) || uri.getAuthority() != null || uri.getQuery() != null + || uri.getFragment() != null) { + throw new IOException("Invalid in-memory WAL reference: " + reference); + } + String path = uri.getPath(); + if (path == null || !path.startsWith(referenceRoot())) { + throw new IOException("Invalid in-memory WAL reference: " + reference); + } + String[] parts = path.substring(referenceRoot().length()).split("/", -1); + if (parts.length != 2 || parts[0].isBlank() || parts[1].isBlank()) { + throw new IOException("Invalid in-memory WAL reference: " + reference); + } + UUID.fromString(parts[1]); + } catch (URISyntaxException | IllegalArgumentException e) { + throw new IOException("Invalid in-memory WAL reference: " + reference, e); + } + } + + private String referenceRoot() { + return "/tmp/accumulo-inmemory-wal-" + port + "/wal/"; + } + + private final class MemoryWriter implements Writer { + private final String reference; + private final ByteArrayOutputStream pending = new ByteArrayOutputStream(); + private final OutputStream output = new OutputStream() { + @Override + public void write(int b) throws IOException { + append(new byte[] {(byte) b}, 0, 1); + } + + @Override + public void write(byte[] bytes, int offset, int length) throws IOException { + Objects.checkFromIndexSize(offset, length, bytes.length); + append(bytes, offset, length); + } + + @Override + public void close() throws IOException { + MemoryWriter.this.close(); + } + }; + private long length; + private boolean closed; + + private MemoryWriter(String reference) { + this.reference = reference; + } + + private synchronized void append(byte[] bytes, int offset, int count) throws IOException { + ensureOpen(); + if (count > 0) { + pending.write(bytes, offset, count); + length += count; + } + } + + private synchronized void flushPending() throws IOException { + if (pending.size() > 0) { + byte[] bytes = pending.toByteArray(); + try (var socket = InMemoryWalStore.connect(host, port); + var output = new DataOutputStream(socket.getOutputStream()); + var input = new DataInputStream(socket.getInputStream())) { + InMemoryWalStore.sendRequest(output, InMemoryWalStore.APPEND, reference); + output.writeInt(bytes.length); + output.write(bytes); + output.flush(); + InMemoryWalStore.checkResponse(input); + } + pending.reset(); + } + } + + private void ensureOpen() throws IOException { + if (closed) { + throw new IOException("WAL writer is closed: " + reference); + } + } + + @Override + public String getReference() { + return reference; + } + + @Override + public OutputStream getOutputStream() { + return output; + } + + @Override + public synchronized void sync(Durability durability) throws IOException { + ensureOpen(); + if (durability != Durability.SYNC && durability != Durability.FLUSH) { + throw new IllegalArgumentException("Unexpected WAL sync durability " + durability); + } + flushPending(); + } + + @Override + public synchronized long getLength() throws IOException { + return length; + } + + @Override + public synchronized void close() throws IOException { + if (!closed) { + flushPending(); + closed = true; + } + } + } +} diff --git a/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalStore.java b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalStore.java new file mode 100644 index 00000000000..0f6448be6dc --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalStore.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.wal; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * An in-memory WAL store shared by {@link InMemoryWalProvider} instances in separate + * MiniAccumuloCluster processes. It listens on an available loopback TCP port, keeps WAL contents + * in the test process heap, and provides the host and port options needed to connect providers. + * Start the store before starting the cluster and keep it running until the cluster has stopped. + * Bytes appended by a provider are retained here after its sync or close, so they remain available + * to other provider processes if a TabletServer exits. + */ +public final class InMemoryWalStore implements AutoCloseable { + + static final byte CREATE = 1; + static final byte APPEND = 2; + static final byte READ = 3; + static final byte EXISTS = 4; + static final byte DELETE = 5; + private static final int CONNECT_TIMEOUT_MILLIS = 10_000; + + private final ServerSocket serverSocket; + private final ExecutorService handlers = Executors.newCachedThreadPool(r -> { + Thread thread = new Thread(r, "in-memory-wal-handler"); + thread.setDaemon(true); + return thread; + }); + private final Set connections = ConcurrentHashMap.newKeySet(); + private final Map wals = new ConcurrentHashMap<>(); + private final Thread acceptThread; + private volatile boolean closed; + + private InMemoryWalStore(ServerSocket serverSocket) { + this.serverSocket = serverSocket; + acceptThread = new Thread(this::acceptConnections, "in-memory-wal-acceptor"); + acceptThread.setDaemon(true); + acceptThread.start(); + } + + /** Starts a store listening on an available loopback port. */ + @SuppressFBWarnings(value = "UNENCRYPTED_SERVER_SOCKET", + justification = "This class is only used in tests.") + public static InMemoryWalStore start() throws IOException { + ServerSocket serverSocket = new ServerSocket(); + serverSocket.setReuseAddress(true); + serverSocket.bind(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0)); + return new InMemoryWalStore(serverSocket); + } + + /** Returns provider options that connect to this store. */ + public Map getProviderOptions() { + return Map.of(InMemoryWalProvider.HOST_OPTION, "127.0.0.1", InMemoryWalProvider.PORT_OPTION, + Integer.toString(serverSocket.getLocalPort())); + } + + private void acceptConnections() { + while (!closed) { + try { + Socket socket = serverSocket.accept(); + connections.add(socket); + handlers.execute(() -> handle(socket)); + } catch (IOException e) { + if (!closed) { + close(); + } + } + } + } + + private void handle(Socket socket) { + try (socket; var input = new DataInputStream(socket.getInputStream()); + var output = new DataOutputStream(socket.getOutputStream())) { + byte command = input.readByte(); + String reference = input.readUTF(); + try { + switch (command) { + case CREATE -> { + if (wals.putIfAbsent(reference, new WalData()) != null) { + throw new IOException("WAL already exists: " + reference); + } + output.writeBoolean(true); + } + case APPEND -> { + int length = input.readInt(); + if (length < 0) { + throw new IOException("Invalid append length: " + length); + } + byte[] bytes = input.readNBytes(length); + if (bytes.length != length) { + throw new EOFException("Incomplete WAL append"); + } + getWal(reference).append(bytes); + output.writeBoolean(true); + } + case READ -> { + byte[] bytes = getWal(reference).snapshot(); + output.writeBoolean(true); + output.writeInt(bytes.length); + output.write(bytes); + } + case EXISTS -> { + output.writeBoolean(true); + output.writeBoolean(wals.containsKey(reference)); + } + case DELETE -> { + wals.remove(reference); + output.writeBoolean(true); + } + default -> throw new IOException("Unknown in-memory WAL operation: " + command); + } + output.flush(); + } catch (IOException | RuntimeException e) { + output.writeBoolean(false); + output.writeUTF(e.getMessage() == null ? e.getClass().getName() : e.getMessage()); + output.flush(); + } + } catch (IOException e) { + // The client may have disconnected before the response was sent. + } finally { + connections.remove(socket); + } + } + + private WalData getWal(String reference) throws IOException { + WalData wal = wals.get(reference); + if (wal == null) { + throw new IOException("WAL does not exist: " + reference); + } + return wal; + } + + private static final class WalData { + private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + synchronized void append(byte[] append) { + bytes.writeBytes(append); + } + + synchronized byte[] snapshot() { + return bytes.toByteArray(); + } + } + + @SuppressFBWarnings(value = "UNENCRYPTED_SOCKET", + justification = "This class is only used in tests.") + static Socket connect(String host, int port) throws IOException { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress(host, port), CONNECT_TIMEOUT_MILLIS); + return socket; + } + + static void sendRequest(DataOutputStream output, byte command, String reference) + throws IOException { + output.writeByte(command); + output.writeUTF(reference); + } + + static void checkResponse(DataInputStream input) throws IOException { + if (!input.readBoolean()) { + throw new IOException(input.readUTF()); + } + } + + @Override + public synchronized void close() { + if (!closed) { + closed = true; + try { + serverSocket.close(); + } catch (IOException e) { + // Closing the listener is best effort during teardown. + } + for (Socket socket : connections) { + try { + socket.close(); + } catch (IOException e) { + // Closing active requests is best effort during teardown. + } + } + handlers.shutdownNow(); + if (Thread.currentThread() != acceptThread) { + try { + acceptThread.join(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + } +} diff --git a/test/src/main/resources/accumulo.properties b/test/src/main/resources/accumulo.properties index b63ed5c0617..2a7b4af5260 100644 --- a/test/src/main/resources/accumulo.properties +++ b/test/src/main/resources/accumulo.properties @@ -21,3 +21,5 @@ instance.dfs.dir=${project.build.directory}/instanceTest instance.secret=TEST_SYSTEM_SECRET +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider diff --git a/test/src/test/java/org/apache/accumulo/test/wal/InMemoryWalProviderTest.java b/test/src/test/java/org/apache/accumulo/test/wal/InMemoryWalProviderTest.java new file mode 100644 index 00000000000..b7852b44ba0 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/wal/InMemoryWalProviderTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.wal; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.UUID; + +import org.apache.accumulo.core.client.Durability; +import org.apache.accumulo.core.spi.common.ServiceEnvironment; +import org.apache.accumulo.core.spi.wal.WalProvider; +import org.junit.jupiter.api.Test; + +class InMemoryWalProviderTest { + + @Test + void sharesSyncedAndClosedWalBytesAcrossProviders() throws Exception { + try (var store = InMemoryWalStore.start()) { + WalProvider writerProvider = newProvider(store.getProviderOptions()); + WalProvider readerProvider = newProvider(store.getProviderOptions()); + UUID id = UUID.randomUUID(); + WalProvider.Writer writer = writerProvider.create("tserver:1234", id); + String reference = writer.getReference(); + + assertEquals("file:/tmp/accumulo-inmemory-wal-" + + store.getProviderOptions().get(InMemoryWalProvider.PORT_OPTION) + "/wal/tserver+1234/" + + id, reference); + assertTrue(readerProvider.exists(reference)); + + writer.getOutputStream().write("header-and-synced-data".getBytes(UTF_8)); + assertEquals(22, writer.getLength()); + writer.sync(Durability.FLUSH); + try (InputStream input = readerProvider.open(reference)) { + assertArrayEquals("header-and-synced-data".getBytes(UTF_8), input.readAllBytes()); + } + + writer.getOutputStream().write("-tail".getBytes(UTF_8)); + writer.close(); + writer.close(); + try (InputStream input = readerProvider.open(reference)) { + assertArrayEquals("header-and-synced-data-tail".getBytes(UTF_8), input.readAllBytes()); + } + assertEquals(27, writer.getLength()); + + readerProvider.delete(reference); + readerProvider.delete(reference); + assertFalse(writerProvider.exists(reference)); + assertThrows(IOException.class, () -> readerProvider.open(reference)); + assertThrows(IOException.class, () -> writer.getOutputStream().write(1)); + } + } + + @Test + void rejectsInvalidProviderOptionsAndReferences() throws Exception { + InMemoryWalProvider uninitializedProvider = new InMemoryWalProvider(); + assertThrows(IllegalArgumentException.class, + () -> uninitializedProvider.init(initParameters(Map.of("host", "127.0.0.1")))); + + try (var store = InMemoryWalStore.start()) { + WalProvider provider = newProvider(store.getProviderOptions()); + assertThrows(IOException.class, () -> provider.exists("file:///tmp/not-a-wal")); + } + } + + @Test + void rejectsLogDurabilitySyncRequests() throws Exception { + try (var store = InMemoryWalStore.start()) { + WalProvider provider = newProvider(store.getProviderOptions()); + try (WalProvider.Writer writer = provider.create("tserver:1234", UUID.randomUUID())) { + assertThrows(IllegalArgumentException.class, () -> writer.sync(Durability.LOG)); + } + } + } + + private static WalProvider newProvider(Map options) { + InMemoryWalProvider provider = new InMemoryWalProvider(); + provider.init(initParameters(options)); + return provider; + } + + private static WalProvider.InitParameters initParameters(Map options) { + return new WalProvider.InitParameters() { + @Override + public ServiceEnvironment getServiceEnvironment() { + return null; + } + + @Override + public Map getOptions() { + return options; + } + }; + } +} From 1be2c1909c76fa8d0c343e8e065933bf188706a3 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Mon, 19 Oct 2026 16:28:08 +0000 Subject: [PATCH 2/8] Minor updates from review --- assemble/conf/accumulo.properties | 8 +++----- .../java/org/apache/accumulo/core/conf/Property.java | 10 ++++------ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/assemble/conf/accumulo.properties b/assemble/conf/accumulo.properties index 5a9532f1d02..9bca9f94da5 100644 --- a/assemble/conf/accumulo.properties +++ b/assemble/conf/accumulo.properties @@ -23,11 +23,9 @@ ## Sets location in HDFS where Accumulo will store data instance.volumes=hdfs://localhost:8020/accumulo -## Configure the WAL provider implementation consistently on every server -## For the built-in filesystem-backed implementation, configure: -## org.apache.accumulo.server.log.DfsWalProvider -# instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} -# instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider +# Configure the WAL provider implementation +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{}} +instance.wal.provider=org.apache.accumulo.server.log.DfsWalProvider ## Sets location of Zookeepers instance.zookeeper.host=localhost:2181 diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index aafef25bef4..b3ca5746db8 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -261,16 +261,14 @@ temporary files (for example, when creating a pre-split table). \ + "org.apache.accumulo.core.spi.crypto.CryptoFactory interface.", "2.1.0"), INSTANCE_WAL_PROVIDERS("instance.wal.providers", "{}", PropertyType.JSON, - "Required JSON object mapping fully qualified class names implementing " - + "org.apache.accumulo.core.spi.wal.WalProvider to string-valued option maps. Use an " + "Map of WAL provider class names and their configuration. Keys are class names that implement " + + "org.apache.accumulo.core.spi.wal.WalProvider. Values are string-valued option maps. Use an " + "empty map when a provider has no options. All configured providers are validated at startup, must be " + "available to Managers, TabletServers, ScanServers, and GC, and must include providers " - + "referenced by existing WALs. This must be consistent across the instance.", + + "referenced by existing WALs.", "4.0.0"), INSTANCE_WAL_PROVIDER("instance.wal.provider", "", PropertyType.CLASSNAME, - "Required. The class implementing org.apache.accumulo.core.spi.wal.WalProvider used to create new " - + "WALs. It must also be listed in instance.wal.providers and must be consistent " - + "across the instance.", + "Class implementing org.apache.accumulo.core.spi.wal.WalProvider, will be used to create new WALs.", "4.0.0"), // general properties GENERAL_PREFIX("general.", null, PropertyType.PREFIX, From 5d4a8f3df6b7655cd9ef7fc885e873d66d36217c Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Wed, 21 Oct 2026 15:43:56 +0000 Subject: [PATCH 3/8] Removed overloaded WalProvider.init method --- WAL-SPI.md | 16 +++++++--------- .../accumulo/core/spi/wal/WalProvider.java | 19 ++++--------------- .../accumulo/server/log/DfsWalProvider.java | 4 ++-- 3 files changed, 13 insertions(+), 26 deletions(-) diff --git a/WAL-SPI.md b/WAL-SPI.md index 1f872a67f69..feeb4447c33 100644 --- a/WAL-SPI.md +++ b/WAL-SPI.md @@ -53,8 +53,6 @@ Each implementation needs a public no-argument constructor and a stable `getId() `[A-Za-z0-9]+`. The ID `dfs` is reserved for `DfsWalProvider` and legacy filesystem WALs. The [registry][registry] loads each configured class, passes its options to `init(InitParameters)` once per server process, rejects duplicate provider IDs, and selects the current class by class name. The -default `init(InitParameters)` implementation calls `init(ServiceEnvironment)`, preserving providers -that implement the older initialization method. A [memoized registry][context] is accessed by the Manager, TabletServers, ScanServers, and GC. The [Manager][manager] also checks WAL IDs in tablet metadata and ZooKeeper markers against the configured IDs at startup. @@ -89,13 +87,13 @@ raw WAL format or the sorted-log recovery process. [`WalProvider`][spi] exposes: -| Method | Use | -| ------------------------------- | ----------------------------------------------------- | -| `create(serverAddress, UUID)` | Exclusively create a WAL and return its `Writer`. | -| `open(reference)` | Read a WAL from byte zero for sorting/recovery. | -| `exists(reference)` | Check a WAL, including while it may still be written. | -| `delete(reference)` | Remove an unreferenced WAL; should be idempotent. | -| `getId()` / `init(environment)` | Supply the persisted ID and initialize the provider. | +| Method | Use | +| ----------------------------------- | ----------------------------------------------------- | +| `create(serverAddress, UUID)` | Exclusively create a WAL and return its `Writer`. | +| `open(reference)` | Read a WAL from byte zero for sorting/recovery. | +| `exists(reference)` | Check a WAL, including while it may still be written. | +| `delete(reference)` | Remove an unreferenced WAL; should be idempotent. | +| `getId()` / `init(InitParameters)` | Supply the persisted ID and initialize the provider. | The nested `Writer` supplies `getReference()`, `getOutputStream()`, `sync(Durability)`, `getLength()`, and `close()`. It can optionally report `getCurrentReplication()` and diff --git a/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java index 842c3dca1b2..689e1e390df 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java @@ -60,25 +60,14 @@ public interface WalProvider { String getId(); /** - * Initializes the provider once after construction. Implementations may retain the environment - * for accessing system configuration and must not assume that properties change during this - * process lifetime. Throwing an exception prevents the server process from starting. - * - * @param environment system configuration and Accumulo service information - */ - default void init(ServiceEnvironment environment) {} - - /** - * Initializes this provider with its configured options. The default implementation calls - * {@link #init(ServiceEnvironment)} for compatibility with providers that only implement that - * method. + * Initializes this provider with its configured options. Implementations may retain the service + * environment for accessing system configuration and must not assume that properties change + * during this process lifetime. Throwing an exception prevents the server process from starting. * * @param params the service environment and options configured for this provider * @since 4.0.0 */ - default void init(InitParameters params) { - init(params.getServiceEnvironment()); - } + default void init(InitParameters params) {} /** Parameters passed to {@link #init(InitParameters)}. */ interface InitParameters { diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java index 096c978b215..d78bdad2773 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java @@ -25,7 +25,6 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Durability; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.spi.common.ServiceEnvironment; import org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment; import org.apache.accumulo.core.spi.wal.WalProvider; import org.apache.accumulo.server.ServerContext; @@ -51,7 +50,8 @@ public String getId() { } @Override - public void init(ServiceEnvironment environment) { + public void init(InitParameters params) { + var environment = params.getServiceEnvironment(); if (!(environment instanceof ServiceEnvironmentImpl serviceEnvironment)) { throw new IllegalArgumentException("DfsWalProvider requires the server service environment"); } From 9c40d7abc675d94b10402fe7bf0782b6ded544b2 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Thu, 12 Nov 2026 20:19:57 +0000 Subject: [PATCH 4/8] Resolve dependencies Assisted-By: OpenAI GPT-6.1 Sol --- .../core/tabletserver/log/LogEntry.java | 29 ++------ .../metadata/schema/TabletMetadataTest.java | 45 ++++-------- .../core/tabletserver/log/LogEntryTest.java | 30 ++++---- .../apache/accumulo/server/fs/VolumeUtil.java | 6 -- .../accumulo/server/log/WalStateManager.java | 6 +- .../manager/state/TabletManagementTest.java | 14 ++-- .../manager/upgrade/Upgrader11to12.java | 62 ++++++---------- .../tableOps/merge/MergeTabletsTest.java | 8 +-- .../tableOps/split/UpdateTabletsTest.java | 8 +-- .../manager/upgrade/Upgrader11to12Test.java | 23 +++--- .../accumulo/tserver/log/DfsLogger.java | 56 +-------------- .../tserver/log/TabletServerLogger.java | 5 -- .../accumulo/tserver/WalRemovalOrderTest.java | 7 +- .../tserver/log/RecoveryLogsIteratorTest.java | 3 +- .../commands/ListTabletsCommandTest.java | 13 ++-- .../MissingWalHeaderCompletesRecoveryIT.java | 24 ++----- .../test/fate/ManagerRepoIT_SimpleSuite.java | 7 +- .../functional/AmpleConditionalWriterIT.java | 72 +++++-------------- .../TabletManagementIteratorIT.java | 11 ++- 19 files changed, 115 insertions(+), 314 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java b/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java index 90b58091536..b66ca1c4845 100644 --- a/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java +++ b/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java @@ -27,11 +27,8 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; -<<<<<<< HEAD import org.apache.accumulo.core.spi.wal.WalProvider; -======= import org.apache.hadoop.fs.Path; ->>>>>>> main import org.apache.hadoop.io.Text; import org.checkerframework.checker.nullness.qual.NonNull; @@ -41,25 +38,17 @@ public final class LogEntry { -<<<<<<< HEAD - private final String path; - private final String providerId; -======= private static final String LOG_PATH = "%s" + Path.SEPARATOR + Constants.WAL_DIR + Path.SEPARATOR + "%s" + Path.SEPARATOR + "%s"; private final Path path; ->>>>>>> main + private final String providerId; private final HostAndPort tserver; private final UUID uniqueId; private final Text columnQualifier; -<<<<<<< HEAD - private LogEntry(String path, String providerId, HostAndPort tserver, UUID uniqueId, + private LogEntry(Path path, String providerId, HostAndPort tserver, UUID uniqueId, Text columnQualifier) { -======= - private LogEntry(Path path, HostAndPort tserver, UUID uniqueId, Text columnQualifier) { ->>>>>>> main this.path = path; this.providerId = providerId; this.tserver = tserver; @@ -68,15 +57,15 @@ private LogEntry(Path path, HostAndPort tserver, UUID uniqueId, Text columnQuali } public static LogEntry withSwitchedVolume(LogEntry entry, Path newPath) { - return new LogEntry(newPath, entry.getTServer(), entry.getUniqueID(), - entry.getColumnQualifier()); + return new LogEntry(validatePath(newPath), entry.getProviderId(), entry.getTServer(), + entry.getUniqueID(), entry.getColumnQualifier()); } - public static LogEntry forServer(String volume, HostAndPort address) { + public static LogEntry forServer(String providerId, String volume, HostAndPort address) { UUID filename = UUID.randomUUID(); String addressForFilename = address.toString().replace(':', '+'); String filePath = LOG_PATH.formatted(volume, addressForFilename, filename); - return new LogEntry(validatePath(new Path(filePath)), address, filename, null); + return fromPath(providerId, filePath); } /** @@ -132,11 +121,8 @@ private static LogEntry validatedLogEntry(String path, String providerId, Text c throw new IllegalArgumentException(badUuidMsg); } -<<<<<<< HEAD - return new LogEntry(path, providerId, tserver, uuid, columnQualifier); -======= Path logPath = validatePath(new Path(path)); - return new LogEntry(logPath, tserver, uuid, columnQualifier); + return new LogEntry(logPath, providerId, tserver, uuid, columnQualifier); } private static Path validatePath(Path path) { @@ -145,7 +131,6 @@ private static Path validatePath(Path path) { "Invalid path format. The path must be an absolute URI: " + path); } return path; ->>>>>>> main } /** diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java index d93bb7b54fd..d37c0606db1 100644 --- a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java +++ b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java @@ -171,15 +171,11 @@ public void testAllColumns() { mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555"); -<<<<<<< HEAD - LogEntry le1 = LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()); + LogEntry le1 = + LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); le1.addToMutation(mutation); - LogEntry le2 = LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()); -======= - LogEntry le1 = LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); - le1.addToMutation(mutation); - LogEntry le2 = LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); ->>>>>>> main + LogEntry le2 = + LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); le2.addToMutation(mutation); StoredTabletFile sf1 = StoredTabletFile.of(new Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf")); @@ -521,12 +517,8 @@ public void testTmBuilderImmutable() { assertThrows(UnsupportedOperationException.class, () -> tm.getFilesMap().put(stf, new DataFileValue(0, 0, 0))); assertTrue(tm.getLogs().isEmpty()); - assertThrows(UnsupportedOperationException.class, () -> tm.getLogs() -<<<<<<< HEAD - .add(LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()))); -======= - .add(LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()))); ->>>>>>> main + assertThrows(UnsupportedOperationException.class, () -> tm.getLogs().add( + LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()))); assertTrue(tm.getScans().isEmpty()); assertThrows(UnsupportedOperationException.class, () -> tm.getScans().add(stf)); assertTrue(tm.getLoaded().isEmpty()); @@ -544,11 +536,7 @@ public void testTmBuilderImmutable() { b.table(TableId.of("4")); b.extCompaction(ecid, ecMeta); b.file(stf, new DataFileValue(0, 0, 0)); -<<<<<<< HEAD - b.log(LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID())); -======= - b.log(LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID())); ->>>>>>> main + b.log(LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID())); b.scan(stf); b.loadedFile(stf, FateId.from(FateInstanceType.USER, UUID.randomUUID())); b.compacted(FateId.from(FateInstanceType.USER, UUID.randomUUID())); @@ -566,12 +554,8 @@ public void testTmBuilderImmutable() { assertThrows(UnsupportedOperationException.class, () -> tm2.getFilesMap().put(stf, new DataFileValue(0, 0, 0))); assertEquals(1, tm2.getLogs().size()); - assertThrows(UnsupportedOperationException.class, () -> tm2.getLogs() -<<<<<<< HEAD - .add(LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()))); -======= - .add(LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()))); ->>>>>>> main + assertThrows(UnsupportedOperationException.class, () -> tm2.getLogs().add( + LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()))); assertEquals(1, tm2.getScans().size()); assertThrows(UnsupportedOperationException.class, () -> tm2.getScans().add(stf)); assertEquals(1, tm2.getLoaded().size()); @@ -847,13 +831,10 @@ public void testBuilder() { new CompactionMetadata(Set.of(sf1, sf2), rf1, "cid1", CompactionKind.USER, (short) 3, ResourceGroupId.of("Q1"), true, FateId.from(type, UUID.randomUUID())); -<<<<<<< HEAD - LogEntry le1 = LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()); - LogEntry le2 = LogEntry.fromPath("example", "localhost+8020/" + UUID.randomUUID()); -======= - LogEntry le1 = LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); - LogEntry le2 = LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); ->>>>>>> main + LogEntry le1 = + LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); + LogEntry le2 = + LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); FateId selFilesFateId = FateId.from(type, UUID.randomUUID()); SelectedFiles selFiles = new SelectedFiles(Set.of(sf1, sf4), false, selFilesFateId, diff --git a/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java b/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java index 05c668b3153..6e204c1ec56 100644 --- a/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java +++ b/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java @@ -31,6 +31,7 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; @@ -63,7 +64,6 @@ public void testFromMetadata() { } @Test -<<<<<<< HEAD public void testProviderIdentityFromMetadata() { var entry = LogEntry.fromPath("example", validPath); var qualifier = entry.getColumnQualifier(); @@ -71,7 +71,7 @@ public void testProviderIdentityFromMetadata() { new Key("1<", LogColumnFamily.STR_NAME, qualifier.toString()), new Value())); assertEquals("example", restored.getProviderId()); - assertEquals(validPath, restored.getPath()); + assertEquals(new Path(validPath), restored.getPath()); assertEquals(entry, restored); assertEquals(PROVIDER_ID, LogEntry.fromPath(PROVIDER_ID, validPath).getProviderId()); } @@ -88,9 +88,11 @@ public void testProviderIdMustBeAlphanumeric() { assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath("provider-id", validPath)); assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath("provider_id", validPath)); assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath("provider.id", validPath)); -======= + } + + @Test public void testForServer() { - var logEntry = LogEntry.forServer("viewfs:/a/accumulo", + var logEntry = LogEntry.forServer(PROVIDER_ID, "viewfs:/a/accumulo", HostAndPort.fromString(validHost.replace('+', ':'))); assertTrue(logEntry.getPath().toUri().getScheme() != null); @@ -107,9 +109,8 @@ public void testForServer() { @Test public void testForServerRequiresQualifiedVolume() { - assertThrows(IllegalArgumentException.class, - () -> LogEntry.forServer("/accumulo", HostAndPort.fromString("localhost:9997"))); ->>>>>>> main + assertThrows(IllegalArgumentException.class, () -> LogEntry.forServer(PROVIDER_ID, "/accumulo", + HostAndPort.fromString("localhost:9997"))); } // helper for testing build from constructor or from metadata @@ -143,17 +144,17 @@ public void testValidPaths() { var validPath1 = "file:///accumulo/wal/" + validHost + "/" + validUUID; var validPath2 = "hdfs://nn1/accumulo/wal/" + validHost + "/" + validUUID; -<<<<<<< HEAD - Stream.of(validPath1, validPath2, validPath3) - .forEach(s -> assertDoesNotThrow(() -> LogEntry.fromPath(PROVIDER_ID, s))); -======= - Stream.of(validPath1, validPath2).forEach(s -> assertDoesNotThrow(() -> LogEntry.fromPath(s))); + for (String path : new String[] {validPath1, validPath2}) { + assertDoesNotThrow(() -> LogEntry.fromPath(PROVIDER_ID, path)); + } } @Test public void testRelativePathsAreInvalid() { - Stream.of(validHost + "/" + validUUID, "file:accumulo/wal/" + validHost + "/" + validUUID) - .forEach(s -> assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath(s))); + for (String path : new String[] {validHost + "/" + validUUID, + "file:accumulo/wal/" + validHost + "/" + validUUID}) { + assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath(PROVIDER_ID, path)); + } } @Test @@ -162,7 +163,6 @@ public void testRelativeMetadataPathIsInvalid() { var entry = new SimpleImmutableEntry( new Key("1<", LogColumnFamily.STR_NAME, qualifier), null); assertThrows(IllegalArgumentException.class, () -> LogEntry.fromMetaWalEntry(entry)); ->>>>>>> main } @Test diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java index 79a62da04d4..63c8db1477a 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java @@ -81,17 +81,11 @@ public static Path switchVolume(Path path, FileType ft, Map replaceme } public static LogEntry switchVolume(LogEntry le, Map replacements) { -<<<<<<< HEAD if (!le.getProviderId().equals(DfsWalProvider.PROVIDER_ID)) { return null; } - Path switchedPath = switchVolume(new Path(le.getPath()), FileType.WAL, replacements); - return switchedPath == null ? null - : LogEntry.fromPath(le.getProviderId(), switchedPath.toString()); -======= Path switchedPath = switchVolume(le.getPath(), FileType.WAL, replacements); return switchedPath == null ? null : LogEntry.withSwitchedVolume(le, switchedPath); ->>>>>>> main } public static boolean needsVolumeReplacement(final Map replacements, diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java index eaca6c4ebd8..34c96b47711 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java @@ -136,7 +136,7 @@ public void addNewWalMarker(TServerInstance tsi, Path path) throws WalMarkerExce } public void addNewWalMarker(TServerInstance tsi, LogEntry entry) throws WalMarkerException { - updateState(tsi, entry.getProviderId(), new Path(entry.getPath()), WalState.OPEN); + updateState(tsi, entry.getProviderId(), entry.getPath(), WalState.OPEN); } private void updateState(TServerInstance tsi, Path path, WalState state) @@ -168,11 +168,11 @@ public void walUnreferenced(TServerInstance tsi, Path path) throws WalMarkerExce } public void walUnreferenced(TServerInstance tsi, LogEntry entry) throws WalMarkerException { - updateState(tsi, entry.getProviderId(), new Path(entry.getPath()), WalState.UNREFERENCED); + updateState(tsi, entry.getProviderId(), entry.getPath(), WalState.UNREFERENCED); } public void closeWal(TServerInstance tsi, LogEntry entry) throws WalMarkerException { - closeWal(tsi, new Path(entry.getPath()), entry.getProviderId()); + closeWal(tsi, entry.getPath(), entry.getProviderId()); } public static WalStatePath parse(byte[] data) { diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java index e95496901c4..98db643f997 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java @@ -109,17 +109,11 @@ private SortedMap createMetadataEntryKV(KeyExtent extent) { mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555"); -<<<<<<< HEAD - LogEntry le1 = - LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); + LogEntry le1 = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, + "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); le1.addToMutation(mutation); - LogEntry le2 = - LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); -======= - LogEntry le1 = LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); - le1.addToMutation(mutation); - LogEntry le2 = LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); ->>>>>>> main + LogEntry le2 = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, + "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); le2.addToMutation(mutation); StoredTabletFile sf1 = diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java index 93ac7d3a006..3d03ce9cb89 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java @@ -88,11 +88,8 @@ import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.core.schema.Section; import org.apache.accumulo.core.security.Authorizations; -<<<<<<< HEAD import org.apache.accumulo.core.spi.wal.WalProvider; -======= import org.apache.accumulo.core.tabletserver.log.LogEntry; ->>>>>>> main import org.apache.accumulo.core.util.Encoding; import org.apache.accumulo.core.util.TextUtil; import org.apache.accumulo.core.util.UtilWaitThread; @@ -234,11 +231,7 @@ public static String getRowPrefix() { @VisibleForTesting static final Set UPGRADE_FAMILIES = Set.of(DataFileColumnFamily.NAME, CHOPPED, -<<<<<<< HEAD ExternalCompactionColumnFamily.NAME, ScanFileColumnFamily.NAME, LogColumnFamily.NAME); -======= - ExternalCompactionColumnFamily.NAME, LogColumnFamily.NAME, ScanFileColumnFamily.NAME); ->>>>>>> main @VisibleForTesting static final String ZNAMESPACE_NAME = "/name"; @@ -841,31 +834,30 @@ static void upgradeDataFileCF(final Key key, final Value value, final Mutation m private static void upgradeWalEntry(Entry entry, Value value, Mutation mutation, Collection volumes) throws IOException { + Text qualifier = entry.getKey().getColumnQualifier(); + String qualifierString = qualifier.toString(); + int separator = qualifierString.indexOf('/'); + Preconditions.checkArgument(separator > 0, "Malformed write-ahead log %s", qualifierString); + String providerId = qualifierString.substring(0, separator); + if (providerId.equals("-")) { + providerId = DfsWalProvider.PROVIDER_ID; + } + String reference = qualifierString.substring(separator + 1); + LogEntry logEntry; try { - LogEntry.fromMetaWalEntry(entry); - return; + logEntry = LogEntry.fromPath(providerId, reference); } catch (IllegalArgumentException e) { - if (e.getMessage().startsWith("Invalid path format. The path must be an absolute URI: ")) { - // The wal entry passed all of the other checks, except that it's not an absolute - // URI. - Text qualifier = entry.getKey().getColumnQualifier(); - String qualifierString = qualifier.toString(); - int separator = qualifierString.indexOf('/'); - if (separator < 0) { - throw new IllegalArgumentException("Malformed write-ahead log " + qualifierString); - } - - String oldPath = qualifierString.substring(separator + 1); - Path qualifiedPath = qualifyWalPath(new Path(oldPath), volumes); - // Validate the expected WAL path structure before writing an upgraded reference. - LogEntry.fromPath(qualifiedPath.toString()); - - Text newQualifier = new Text(qualifierString.substring(0, separator + 1) + qualifiedPath); - mutation.at().family(LogColumnFamily.NAME).qualifier(newQualifier).put(value); - mutation.at().family(LogColumnFamily.NAME).qualifier(qualifier).delete(); - } else { + if (!providerId.equals(DfsWalProvider.PROVIDER_ID) || !e.getMessage() + .startsWith("Invalid path format. The path must be an absolute URI: ")) { throw e; } + Path qualifiedPath = qualifyWalPath(new Path(reference), volumes); + logEntry = LogEntry.fromPath(providerId, qualifiedPath.toString()); + } + Text newQualifier = logEntry.getColumnQualifier(); + if (!newQualifier.equals(qualifier)) { + mutation.at().family(LogColumnFamily.NAME).qualifier(newQualifier).put(value); + mutation.at().family(LogColumnFamily.NAME).qualifier(qualifier).delete(); } } @@ -953,8 +945,6 @@ void processReferences(MutationWriter batchWriter, Iterable tableName, key.getRow()); update.at().family(ExternalCompactionColumnFamily.NAME) .qualifier(key.getColumnQualifier()).delete(); - } else if (family.equals(LogColumnFamily.NAME)) { - upgradeWalReference(key, value, update); } else { throw new IllegalStateException("Processing: " + tableName + " Received unexpected column family processing references: " + family); @@ -972,16 +962,6 @@ void processReferences(MutationWriter batchWriter, Iterable } } - @VisibleForTesting - static void upgradeWalReference(Key key, Value value, Mutation mutation) { - String qualifier = key.getColumnQualifier().toString(); - if (qualifier.startsWith("-/")) { - Text upgradedQualifier = new Text(DfsWalProvider.PROVIDER_ID + qualifier.substring(1)); - mutation.at().family(LogColumnFamily.NAME).qualifier(upgradedQualifier).put(value); - mutation.at().family(LogColumnFamily.NAME).qualifier(key.getColumnQualifier()).delete(); - } - } - private static void verifyNoLegacyWalReference(Key key, String source) { if (key.getColumnFamily().equals(LogColumnFamily.NAME) && key.getColumnQualifier().toString().startsWith("-/")) { @@ -1074,6 +1054,8 @@ void updateRootTabletFileReferences(ServerContext context) { zrw.overwritePersistentData(ZROOT_TABLET, rtm.toJson().getBytes(UTF_8), stat.getVersion()); LOG.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson()); } + rtm.getKeyValues() + .forEach(entry -> verifyNoLegacyWalReference(entry.getKey(), "root tablet")); } catch (KeeperException | InterruptedException e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java index 0da04f0503c..78850df43e3 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/merge/MergeTabletsTest.java @@ -331,12 +331,8 @@ public void testUnexpectedColumns() { testUnexpectedColumn(tmb -> tmb.putOperation(otherOpid), "had unexpected opid", otherOpid.toString()); -<<<<<<< HEAD - var walog = - LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); -======= - var walog = LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); ->>>>>>> main + var walog = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, + "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); testUnexpectedColumn(tmb -> tmb.putWal(walog), "has unexpected walogs 1"); FateId ucfid1 = otherFateId; diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java index 01ca2dbb007..912361c85ab 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/split/UpdateTabletsTest.java @@ -467,12 +467,8 @@ public void testErrors() throws Exception { assertTrue(e.getMessage().contains(opid2.toString())); // Test splitting a tablet with walogs -<<<<<<< HEAD - var walog = - LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); -======= - var walog = LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); ->>>>>>> main + var walog = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, + "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); var tablet4 = TabletMetadata.builder(origExtent).putOperation(opid).putWal(walog) .build(ColumnType.LOCATION); e = assertThrows(IllegalStateException.class, () -> testError(origExtent, tablet4, fateId)); diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java index dc52ac4a5fb..e4b02f8d81d 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java @@ -85,14 +85,11 @@ import org.apache.accumulo.server.conf.store.SystemPropKey; import org.apache.accumulo.server.conf.store.TablePropKey; import org.apache.accumulo.server.conf.store.impl.ZooPropStore; -<<<<<<< HEAD +import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.WalStateManager; -======= -import org.apache.accumulo.server.fs.VolumeManager; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; ->>>>>>> main import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.zookeeper.KeeperException; @@ -200,22 +197,21 @@ public void processReferencesTest() { var u2 = mutations.get(1); LOG.info("c:{}", u2.prettyPrint()); - // 1 add, 2 delete - assertEquals(3, u2.getUpdates().size()); - assertEquals(2, u2.getUpdates().stream().filter(ColumnUpdate::isDeleted).count()); + // 2 add, 3 delete (including the WAL provider ID conversion) + assertEquals(5, u2.getUpdates().size()); + assertEquals(3, u2.getUpdates().stream().filter(ColumnUpdate::isDeleted).count()); assertTrue(UPGRADE_FAMILIES.contains(LogColumnFamily.NAME)); } @Test -<<<<<<< HEAD void upgradeWalReferenceTest() { String path = "hdfs://nn/accumulo/wal/host+1234/11111111-1111-1111-1111-111111111111"; Key key = Key.builder().row("123").family(LogColumnFamily.NAME).qualifier("-/" + path).build(); Value value = new Value(); ArrayList mutations = new ArrayList<>(); new Upgrader11to12().processReferences(mutations::add, List.of(Map.entry(key, value)), - "accumulo.metadata"); + "accumulo.metadata", List.of()); assertTrue(UPGRADE_FAMILIES.contains(LogColumnFamily.NAME)); assertEquals(1, mutations.size()); @@ -232,7 +228,7 @@ void upgradeWalReferenceTest() { .qualifier(DfsWalProvider.PROVIDER_ID + "/" + path).build(); ArrayList retryMutations = new ArrayList<>(); new Upgrader11to12().processReferences(retryMutations::add, - List.of(Map.entry(upgradedKey, value)), "accumulo.metadata"); + List.of(Map.entry(upgradedKey, value)), "accumulo.metadata", List.of()); assertTrue(retryMutations.isEmpty()); } @@ -280,7 +276,9 @@ void upgradeWalMarkersTest() throws Exception { replay(context, zk, zrw); new Upgrader11to12().upgradeWalMarkers(context); verify(context, zk, zrw); -======= + } + + @Test public void processReferencesQualifiesWalPathsUsingVolume(@TempDir java.nio.file.Path tempDir) throws Exception { Configuration conf = new Configuration(); @@ -304,7 +302,7 @@ public void processReferencesQualifiesWalPathsUsingVolume(@TempDir java.nio.file .qualifier(oldQualifier).build(); scanData.put(walKey, new Value("wal-value")); oldQualifiers.add(oldQualifier); - expectedQualifiers.add("-/" + existingWal); + expectedQualifiers.add(DfsWalProvider.PROVIDER_ID + "/" + existingWal); } ArrayList mutations = new ArrayList<>(); @@ -357,7 +355,6 @@ public void processReferencesRejectsAmbiguousRelativeWalPath(@TempDir java.nio.f assertTrue(error.getMessage().contains("accumulo.metadata tablet 123")); assertTrue(error.getCause().getMessage().contains("exactly one configured volume")); } ->>>>>>> main } @Test diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java index 0fdf9721fbd..2574028372f 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicLong; @@ -73,7 +74,6 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; -import com.google.common.net.HostAndPort; /** * Wrap a connection to a logger. @@ -305,7 +305,6 @@ public int hashCode() { /** * Create a new DfsLogger with the provided characteristics. */ -<<<<<<< HEAD public static DfsLogger createNew(ServerContext context, WalProvider provider, AtomicLong syncCounter, AtomicLong flushCounter, String address) throws IOException { @@ -330,21 +329,6 @@ public static DfsLogger createNew(ServerContext context, WalProvider provider, } throw e; } -======= - public static DfsLogger createNew(ServerContext context, AtomicLong syncCounter, - AtomicLong flushCounter, HostAndPort address) throws IOException { - - var chooserEnv = - new VolumeChooserEnvironmentImpl(VolumeChooserEnvironment.Scope.LOGGER, context); - String logVolume = context.getVolumeManager().choose(chooserEnv, context.getBaseUris()); - - LogEntry log = LogEntry.forServer(logVolume, address); - DfsLogger dfsLogger = new DfsLogger(log); - long slowFlushMillis = - context.getConfiguration().getTimeInMillis(Property.TSERV_SLOW_FLUSH_MILLIS); - dfsLogger.open(context, syncCounter, flushCounter, slowFlushMillis); - return dfsLogger; ->>>>>>> main } /** @@ -415,50 +399,16 @@ public static DataInputStream getDecryptingStream(InputStream input, CryptoServi * exception is thrown from this method, it is the callers responsibility to ensure that * {@link #close()} is called to prevent leaking the file handle and/or syncing thread. */ -<<<<<<< HEAD private synchronized void open(ServerContext context, WalProvider.Writer writer, String address, AtomicLong syncCounter, AtomicLong flushCounter, long slowFlushMillis) throws IOException { log.debug("Address is {}", address); -======= - private synchronized void open(ServerContext context, AtomicLong syncCounter, - AtomicLong flushCounter, long slowFlushMillis) throws IOException { - log.debug("Address is {}", logEntry.getTServer()); ->>>>>>> main log.debug("DfsLogger.open() begin"); LoggerOperation op; try { -<<<<<<< HEAD logFile = writer; OutputStream output = logFile.getOutputStream(); -======= - Path logfilePath = logEntry.getPath(); - short replication = (short) serverConf.getCount(Property.TSERV_WAL_REPLICATION); - if (replication == 0) { - replication = fs.getDefaultReplication(logfilePath); - } - long blockSize = getWalBlockSize(serverConf); - if (serverConf.getBoolean(Property.TSERV_WAL_SYNC)) { - logFile = fs.createSyncable(logfilePath, 0, replication, blockSize); - } else { - logFile = fs.create(logfilePath, true, 0, replication, blockSize); - } - - // Tell the DataNode that the write ahead log does not need to be cached in the OS page cache - try { - logFile.setDropBehind(Boolean.TRUE); - } catch (UnsupportedOperationException e) { - log.debug("setDropBehind writes not enabled for wal file: {}", logFile); - } catch (IOException e) { - log.debug("IOException setting drop behind for file: {}, msg: {}", logFile, e.getMessage()); - } - - // check again that logfile can be sync'd - if (!fs.canSyncAndFlush(logfilePath)) { - log.warn("sync not supported for log file {}. Data loss may occur.", logEntry.getPath()); - } ->>>>>>> main // Initialize the log file with a header and its encryption CryptoEnvironment env = new CryptoEnvironmentImpl(Scope.WAL); @@ -467,11 +417,7 @@ private synchronized void open(ServerContext context, AtomicLong syncCounter, output.write(LOG_FILE_HEADER_V4.getBytes(UTF_8)); log.debug("Using {} for encrypting WAL {}", cryptoService.getClass().getSimpleName(), -<<<<<<< HEAD writer.getReference()); -======= - logEntry.getUniqueID()); ->>>>>>> main FileEncrypter encrypter = cryptoService.getFileEncrypter(env); byte[] cryptoParams = encrypter.getDecryptionParameters(); CryptoUtils.writeParams(cryptoParams, new DataOutputStream(output)); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java index 86a498f3fc8..fa9c97603b8 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java @@ -277,14 +277,9 @@ private synchronized void startLogMaker() { final DfsLogger alog; try { -<<<<<<< HEAD alog = DfsLogger.createNew(tserver.getContext(), tserver.getContext().getWalProviderRegistry().getCurrent(), syncCounter, flushCounter, tserver.getAdvertiseAddress().toString()); -======= - alog = DfsLogger.createNew(tserver.getContext(), syncCounter, flushCounter, - tserver.getAdvertiseAddress()); ->>>>>>> main } catch (IOException | RuntimeException e) { log.error("Failed to open WAL", e); try { diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/WalRemovalOrderTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/WalRemovalOrderTest.java index aeaff325d52..3576a4627ab 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/WalRemovalOrderTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/WalRemovalOrderTest.java @@ -35,13 +35,8 @@ public class WalRemovalOrderTest { private DfsLogger mockLogger(String filename) { -<<<<<<< HEAD var mockLogEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, - filename + "+1234/11111111-1111-1111-1111-111111111111"); -======= - var mockLogEntry = LogEntry - .fromPath("file:/accumulo/wal/" + filename + "+1234/11111111-1111-1111-1111-111111111111"); ->>>>>>> main + "file:/accumulo/wal/" + filename + "+1234/11111111-1111-1111-1111-111111111111"); return DfsLogger.fromLogEntry(mockLogEntry); } diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsIteratorTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsIteratorTest.java index f87424325a5..c650087f2ac 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsIteratorTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/RecoveryLogsIteratorTest.java @@ -186,7 +186,8 @@ public void testSingleFile() throws IOException { assertThrows(IOException.class, () -> new RecoveryLogsIterator(context, - Collections.singletonList(ResolvedSortedLog.resolve(LogEntry.fromPath(destPath), fs)), + Collections.singletonList(ResolvedSortedLog + .resolve(LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, destPath), fs)), null, null, false), "Finish marker should not be found for a single file."); } diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java index bc7b6408711..cb489ca3f71 100644 --- a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java +++ b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java @@ -192,15 +192,10 @@ public void mockTest() throws Exception { KeyExtent extent = new KeyExtent(tableId, new Text("d"), null); -<<<<<<< HEAD - LogEntry le1 = - LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); - LogEntry le2 = - LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, "localhost+8020/" + UUID.randomUUID()); -======= - LogEntry le1 = LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); - LogEntry le2 = LogEntry.fromPath("file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); ->>>>>>> main + LogEntry le1 = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, + "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); + LogEntry le2 = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, + "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); TabletMetadata tm1 = TabletMetadata.builder(extent).putTabletAvailability(TabletAvailability.ONDEMAND) diff --git a/test/src/main/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java b/test/src/main/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java index 5ea30f1fadf..12f409c6e07 100644 --- a/test/src/main/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java +++ b/test/src/main/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java @@ -114,9 +114,9 @@ public void testEmptyWalRecoveryCompletes() throws Exception { MiniAccumuloClusterImpl cluster = getCluster(); FileSystem fs = cluster.getFileSystem(); - LogEntry logEntry = - LogEntry.forServer(cluster.getConfig().getAccumuloDir().toPath().toUri().toString(), - HostAndPort.fromParts("127.0.0.1", 12345)); + LogEntry logEntry = LogEntry.forServer(DfsWalProvider.PROVIDER_ID, + cluster.getConfig().getAccumuloDir().toPath().toUri().toString(), + HostAndPort.fromParts("127.0.0.1", 12345)); log.info("Created empty WAL at {}", logEntry.getPath().toUri()); @@ -132,12 +132,6 @@ public void testEmptyWalRecoveryCompletes() throws Exception { TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName)); assertNotNull(tableId, "Table ID was null"); -<<<<<<< HEAD - LogEntry logEntry = - LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, emptyWalog.toUri().toString()); - -======= ->>>>>>> main log.info("Taking {} offline", tableName); client.tableOperations().offline(tableName, true); @@ -170,9 +164,9 @@ public void testPartialHeaderWalRecoveryCompletes() throws Exception { MiniAccumuloClusterImpl cluster = getCluster(); FileSystem fs = cluster.getFileSystem(); - LogEntry logEntry = - LogEntry.forServer(cluster.getConfig().getAccumuloDir().toPath().toUri().toString(), - HostAndPort.fromParts("127.0.0.1", 12345)); + LogEntry logEntry = LogEntry.forServer(DfsWalProvider.PROVIDER_ID, + cluster.getConfig().getAccumuloDir().toPath().toUri().toString(), + HostAndPort.fromParts("127.0.0.1", 12345)); log.info("Created WAL with malformed header at {}", logEntry.getPath().toUri()); // Write half of the header @@ -191,12 +185,6 @@ public void testPartialHeaderWalRecoveryCompletes() throws Exception { TableId tableId = TableId.of(client.tableOperations().tableIdMap().get(tableName)); assertNotNull(tableId, "Table ID was null"); -<<<<<<< HEAD - LogEntry logEntry = - LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, partialHeaderWalog.toUri().toString()); - -======= ->>>>>>> main log.info("Taking {} offline", tableName); client.tableOperations().offline(tableName, true); diff --git a/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java b/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java index ea80c74e1e4..28920c644fe 100644 --- a/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java +++ b/test/src/main/java/org/apache/accumulo/test/fate/ManagerRepoIT_SimpleSuite.java @@ -161,15 +161,10 @@ public void testNoWalsMergeRepos(MergeInfo.Operation operation) throws Exception // Write a WAL to the test metadata and then re-run the repo to check for an error try (TabletsMutator tm = testAmple.mutateTablets()) { -<<<<<<< HEAD - var walFilePath = Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - tm.mutateTablet(extent).putWal(LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath)) -======= tm.mutateTablet(extent) - .putWal(LogEntry.forServer( + .putWal(LogEntry.forServer(DfsWalProvider.PROVIDER_ID, getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), HostAndPort.fromParts("tserver", 8080))) ->>>>>>> main .mutate(); } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java index 737ae5dcb29..ea02c610b35 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java @@ -473,15 +473,9 @@ public void testWALs() { var context = getCluster().getServerContext(); // Test adding a WAL to a tablet and verifying its presence -<<<<<<< HEAD - String walFilePath = - java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - final LogEntry originalLogEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath); -======= - final LogEntry originalLogEntry = - LogEntry.forServer(getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), - HostAndPort.fromParts("tserver", 8080)); ->>>>>>> main + final LogEntry originalLogEntry = LogEntry.forServer(DfsWalProvider.PROVIDER_ID, + getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), + HostAndPort.fromParts("tserver", 8080)); // create a tablet metadata with no write ahead logs var tmEmptySet = TabletMetadata.builder(e1).build(LOGS); // tablet should not have any logs to start with so requireSame with the empty logs should pass @@ -497,15 +491,9 @@ public void testWALs() { "The original LogEntry should be present."); // Test adding another WAL and verifying the update -<<<<<<< HEAD - String walFilePath2 = - java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - LogEntry newLogEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath2); -======= - LogEntry newLogEntry = - LogEntry.forServer(getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), - HostAndPort.fromParts("tserver", 8080)); ->>>>>>> main + LogEntry newLogEntry = LogEntry.forServer(DfsWalProvider.PROVIDER_ID, + getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), + HostAndPort.fromParts("tserver", 8080)); try (var ctmi = context.getAmple().conditionallyMutateTablets()) { ctmi.mutateTablet(e1).requireAbsentOperation().putWal(newLogEntry).submit(tm -> false); assertEquals(Status.ACCEPTED, ctmi.process().get(e1).getStatus()); @@ -516,16 +504,9 @@ public void testWALs() { HashSet actualLogs = new HashSet<>(context.getAmple().readTablet(e1).getLogs()); assertEquals(expectedLogs, actualLogs, "Both original and new LogEntry should be present."); -<<<<<<< HEAD - String walFilePath3 = - java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - LogEntry otherLogEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath3); - -======= - LogEntry otherLogEntry = - LogEntry.forServer(getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), - HostAndPort.fromParts("tserver", 8080)); ->>>>>>> main + LogEntry otherLogEntry = LogEntry.forServer(DfsWalProvider.PROVIDER_ID, + getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), + HostAndPort.fromParts("tserver", 8080)); // create a powerset to ensure all possible subsets fail when using requireSame except the // expected current state Set allLogs = Set.of(originalLogEntry, newLogEntry, otherLogEntry); @@ -1254,15 +1235,9 @@ public void multipleFilters() { "Initially, all tablets should be present"); String server = "server1+8555"; -<<<<<<< HEAD - - String walFilePath = java.nio.file.Path.of(server, UUID.randomUUID().toString()).toString(); - LogEntry wal = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath); -======= - LogEntry wal = - LogEntry.forServer(getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), - HostAndPort.fromParts("server1", 8555)); ->>>>>>> main + LogEntry wal = LogEntry.forServer(DfsWalProvider.PROVIDER_ID, + getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), + HostAndPort.fromParts("server1", 8555)); // add wal compact and flush ID to these tablets final Set tabletsWithWalCompactFlush = Set.of(e1, e2, e3); @@ -1364,15 +1339,9 @@ public void walFilter() { "Initially, all tablets should be present"); // add a wal to e2 -<<<<<<< HEAD - var walFilePath = - java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - var wal = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath); -======= - var wal = - LogEntry.forServer(getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), - HostAndPort.fromParts("tserver", 8080)); ->>>>>>> main + var wal = LogEntry.forServer(DfsWalProvider.PROVIDER_ID, + getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), + HostAndPort.fromParts("tserver", 8080)); try (var ctmi = context.getAmple().conditionallyMutateTablets()) { ctmi.mutateTablet(e2).requireAbsentOperation().putWal(wal).submit(tabletMetadata -> false); assertEquals(Status.ACCEPTED, ctmi.process().get(e2).getStatus()); @@ -1382,14 +1351,9 @@ public void walFilter() { testFilterApplied(context, filter, Set.of(e2), "Only tablets with wals should be returned"); // add wal to tablet e4 -<<<<<<< HEAD - walFilePath = java.nio.file.Path.of("tserver+8080", UUID.randomUUID().toString()).toString(); - wal = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, walFilePath); -======= - wal = - LogEntry.forServer(getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), - HostAndPort.fromParts("tserver", 8080)); ->>>>>>> main + wal = LogEntry.forServer(DfsWalProvider.PROVIDER_ID, + getCluster().getConfig().getAccumuloDir().toPath().toUri().toString(), + HostAndPort.fromParts("tserver", 8080)); try (var ctmi = context.getAmple().conditionallyMutateTablets()) { ctmi.mutateTablet(e4).requireAbsentOperation().putWal(wal).submit(tabletMetadata -> false); assertEquals(Status.ACCEPTED, ctmi.process().get(e4).getStatus()); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java index e00cec5b2d2..49f6100b0d1 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java @@ -674,13 +674,10 @@ private void createLogEntry(AccumuloClient client, String table, String tableNam TableId.of(client.tableOperations().tableIdMap().get(tableNameToModify)); KeyExtent extent = new KeyExtent(tableIdToModify, new Text("some split"), null); Mutation m = new Mutation(extent.toMetaRow()); -<<<<<<< HEAD - String fileName = "file:/accumulo/wal/localhost+9997/" + UUID.randomUUID().toString(); - LogEntry logEntry = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, fileName); -======= - LogEntry logEntry = LogEntry.forServer(((MiniAccumuloClusterImpl) getCluster()).getConfig() - .getAccumuloDir().toPath().toUri().toString(), HostAndPort.fromParts("localhost", 9997)); ->>>>>>> main + LogEntry logEntry = LogEntry.forServer( + DfsWalProvider.PROVIDER_ID, ((MiniAccumuloClusterImpl) getCluster()).getConfig() + .getAccumuloDir().toPath().toUri().toString(), + HostAndPort.fromParts("localhost", 9997)); logEntry.addToMutation(m); try (BatchWriter bw = client.createBatchWriter(table)) { bw.addMutation(m); From dbfeff7946599bce866763c920c7c2c74ddc526b Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Thu, 12 Nov 2026 20:49:17 +0000 Subject: [PATCH 5/8] Addressed PR comments --- .../accumulo/core/spi/wal/WalProvider.java | 14 ++---- .../accumulo/server/log/DfsWalProvider.java | 20 ++++++--- .../accumulo/tserver/log/DfsLogger.java | 45 +++---------------- .../test/wal/InMemoryWalProvider.java | 11 +++++ 4 files changed, 35 insertions(+), 55 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java index 689e1e390df..fe86d9f003c 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java @@ -155,20 +155,14 @@ interface Writer extends AutoCloseable { long getLength() throws IOException; /** - * Returns the current replication level when the provider can report one, or {@code -1} when - * that information is unavailable. + * Returns whether or not the writer is healthy */ - default int getCurrentReplication() throws IOException { - return -1; - } + boolean isHealthy() throws IOException; /** - * Returns a diagnostic description of the current storage pipeline, or an empty array when the - * provider has no pipeline information. + * Returns a String containing implementation relevant information */ - default String[] getPipeline() { - return new String[0]; - } + String getDiagnostics() throws IOException; /** * Closes the WAL and releases all associated resources. Closing must be idempotent because diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java index d78bdad2773..8d6fa46a5d6 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSOutputStream; +import org.apache.hadoop.hdfs.DFSStripedOutputStream; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; /** Filesystem-backed WAL provider. */ @@ -152,23 +153,28 @@ public long getLength() throws IOException { } @Override - public int getCurrentReplication() throws IOException { + public boolean isHealthy() throws IOException { var wrapped = output.getWrappedStream(); - return wrapped instanceof DFSOutputStream dfs ? dfs.getCurrentBlockReplication() : -1; + if (wrapped instanceof DFSOutputStream repl) { + return repl.getCurrentBlockReplication() > 1; + } else if (wrapped instanceof DFSStripedOutputStream) { + return true; + } else { + return false; + } } @Override - public String[] getPipeline() { + public String getDiagnostics() throws IOException { + StringBuilder buf = new StringBuilder(); var wrapped = output.getWrappedStream(); if (wrapped instanceof DFSOutputStream dfs) { DatanodeInfo[] pipeline = dfs.getPipeline(); - String[] names = new String[pipeline.length]; for (int i = 0; i < pipeline.length; i++) { - names[i] = pipeline[i].getName(); + buf.append("name: ").append(pipeline[i].getName()).append("\n"); } - return names; } - return new String[0]; + return buf.toString(); } @Override diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java index 2574028372f..4e8034a487f 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java @@ -124,7 +124,6 @@ public LogHeaderIncompleteException(EOFException cause) { private boolean closed = false; private class LogSyncingTask implements Runnable { - private int expectedReplication = 0; private final AtomicLong syncCounter; private final AtomicLong flushCounter; @@ -182,33 +181,16 @@ public void run() { flushCounter.incrementAndGet(); } } - } catch (IOException | RuntimeException ex) { - fail(work, ex, "synching"); - } - if (timer.hasElapsed(slowFlushDuration)) { - log.info("Slow sync cost: {} ms, current pipeline: {}", timer.elapsed(MILLISECONDS), - Arrays.toString(getPipeLine())); - if (expectedReplication > 0) { - int current = expectedReplication; - try { - current = logFile.getCurrentReplication(); - } catch (IOException e) { - fail(work, e, "getting replication level"); - } - if (current < expectedReplication) { - fail(work, - new IOException( - "replication of " + current + " is less than " + expectedReplication), + if (timer.hasElapsed(slowFlushDuration)) { + log.info("Slow sync cost: {} ms, current information: {}", timer.elapsed(MILLISECONDS), + logFile.getDiagnostics()); + if (!logFile.isHealthy()) { + fail(work, new IOException("WalProvider is reporting it's not healthly"), "replication check"); } } - } - if (expectedReplication == 0) { - try { - expectedReplication = logFile.getCurrentReplication(); - } catch (IOException e) { - fail(work, e, "getting replication level"); - } + } catch (IOException | RuntimeException ex) { + fail(work, ex, "syncing"); } for (LogWork logWork : work) { @@ -641,17 +623,4 @@ public int compareTo(DfsLogger o) { return logEntry.getPath().compareTo(o.logEntry.getPath()); } - /* - * The following method was shamelessly lifted from HBASE-11240 (sans reflection). Thanks HBase! - */ - - /** - * This method gets the pipeline for the current walog. - * - * @return diagnostic storage pipeline - */ - String[] getPipeLine() { - return logFile == null ? new String[0] : logFile.getPipeline(); - } - } diff --git a/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java index eee1b1c817e..d958d5c2c52 100644 --- a/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java +++ b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java @@ -204,6 +204,7 @@ public void write(byte[] bytes, int offset, int length) throws IOException { public void close() throws IOException { MemoryWriter.this.close(); } + }; private long length; private boolean closed; @@ -273,5 +274,15 @@ public synchronized void close() throws IOException { closed = true; } } + + @Override + public boolean isHealthy() throws IOException { + return true; + } + + @Override + public String getDiagnostics() throws IOException { + return ""; + } } } From e4be4309949e0c3781b9f591e1103ba43e6691a2 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Fri, 13 Nov 2026 19:38:26 +0000 Subject: [PATCH 6/8] Changes from review --- .../metadata/schema/TabletMetadataTest.java | 27 ++++++++++--------- .../accumulo/server/log/DfsWalProvider.java | 7 +++-- .../server/log/WalProviderRegistry.java | 2 -- .../accumulo/server/log/WalStateManager.java | 15 ----------- .../manager/recovery/RecoveryManager.java | 8 +++--- .../manager/upgrade/Upgrader11to12.java | 2 +- 6 files changed, 23 insertions(+), 38 deletions(-) diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java index d37c0606db1..0efe70169e6 100644 --- a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java +++ b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java @@ -171,11 +171,11 @@ public void testAllColumns() { mutation.at().family(LastLocationColumnFamily.NAME).qualifier("s000").put("server2:8555"); - LogEntry le1 = - LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); + LogEntry le1 = LogEntry.fromPath("exampleProviderId", + "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); le1.addToMutation(mutation); - LogEntry le2 = - LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); + LogEntry le2 = LogEntry.fromPath("exampleProviderId", + "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); le2.addToMutation(mutation); StoredTabletFile sf1 = StoredTabletFile.of(new Path("hdfs://nn1/acc/tables/1/t-0001/sf1.rf")); @@ -517,8 +517,8 @@ public void testTmBuilderImmutable() { assertThrows(UnsupportedOperationException.class, () -> tm.getFilesMap().put(stf, new DataFileValue(0, 0, 0))); assertTrue(tm.getLogs().isEmpty()); - assertThrows(UnsupportedOperationException.class, () -> tm.getLogs().add( - LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()))); + assertThrows(UnsupportedOperationException.class, () -> tm.getLogs().add(LogEntry + .fromPath("exampleProviderId", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()))); assertTrue(tm.getScans().isEmpty()); assertThrows(UnsupportedOperationException.class, () -> tm.getScans().add(stf)); assertTrue(tm.getLoaded().isEmpty()); @@ -536,7 +536,8 @@ public void testTmBuilderImmutable() { b.table(TableId.of("4")); b.extCompaction(ecid, ecMeta); b.file(stf, new DataFileValue(0, 0, 0)); - b.log(LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID())); + b.log(LogEntry.fromPath("exampleProviderId", + "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID())); b.scan(stf); b.loadedFile(stf, FateId.from(FateInstanceType.USER, UUID.randomUUID())); b.compacted(FateId.from(FateInstanceType.USER, UUID.randomUUID())); @@ -554,8 +555,8 @@ public void testTmBuilderImmutable() { assertThrows(UnsupportedOperationException.class, () -> tm2.getFilesMap().put(stf, new DataFileValue(0, 0, 0))); assertEquals(1, tm2.getLogs().size()); - assertThrows(UnsupportedOperationException.class, () -> tm2.getLogs().add( - LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()))); + assertThrows(UnsupportedOperationException.class, () -> tm2.getLogs().add(LogEntry + .fromPath("exampleProviderId", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()))); assertEquals(1, tm2.getScans().size()); assertThrows(UnsupportedOperationException.class, () -> tm2.getScans().add(stf)); assertEquals(1, tm2.getLoaded().size()); @@ -831,10 +832,10 @@ public void testBuilder() { new CompactionMetadata(Set.of(sf1, sf2), rf1, "cid1", CompactionKind.USER, (short) 3, ResourceGroupId.of("Q1"), true, FateId.from(type, UUID.randomUUID())); - LogEntry le1 = - LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); - LogEntry le2 = - LogEntry.fromPath("example", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); + LogEntry le1 = LogEntry.fromPath("exampleProviderId", + "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); + LogEntry le2 = LogEntry.fromPath("exampleProviderId", + "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); FateId selFilesFateId = FateId.from(type, UUID.randomUUID()); SelectedFiles selFiles = new SelectedFiles(Set.of(sf1, sf4), false, selFilesFateId, diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java index 8d6fa46a5d6..c7f514d49fc 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java @@ -36,10 +36,14 @@ import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.DFSStripedOutputStream; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** Filesystem-backed WAL provider. */ public class DfsWalProvider implements WalProvider { + private static final Logger LOG = LoggerFactory.getLogger(DfsWalProvider.class); + /** Stable provider ID used for WALs stored on HDFS. */ public static final String PROVIDER_ID = "dfs"; @@ -91,8 +95,7 @@ public Writer create(String serverAddress, UUID id) throws IOException { // Drop-behind is a performance hint, not a requirement. } if (!fs.canSyncAndFlush(path)) { - org.slf4j.LoggerFactory.getLogger(DfsWalProvider.class) - .warn("sync not supported for log file {}. Data loss may occur.", logPath); + LOG.warn("sync not supported for log file {}. Data loss may occur.", logPath); } return new DfsWriter(logPath, output); } diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java index e08f4970100..5549c1bffca 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalProviderRegistry.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Set; -import org.apache.accumulo.core.classloader.ClassLoaderUtil; import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.conf.PropertyType; @@ -61,7 +60,6 @@ static void validateProviderId(String id, String className) { public WalProviderRegistry(ServerContext context) { var conf = context.getConfiguration(); - ClassLoaderUtil.initContextFactory(conf); var environment = new ServiceEnvironmentImpl(context); String currentClass = conf.get(Property.INSTANCE_WAL_PROVIDER).trim(); if (currentClass.isEmpty()) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java index 34c96b47711..118ef0f1197 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java @@ -130,20 +130,10 @@ public void initWalMarker(TServerInstance tsi) throws WalMarkerException { } } - // Tablet server opens a new WAL - public void addNewWalMarker(TServerInstance tsi, Path path) throws WalMarkerException { - updateState(tsi, DfsWalProvider.PROVIDER_ID, path, WalState.OPEN); - } - public void addNewWalMarker(TServerInstance tsi, LogEntry entry) throws WalMarkerException { updateState(tsi, entry.getProviderId(), entry.getPath(), WalState.OPEN); } - private void updateState(TServerInstance tsi, Path path, WalState state) - throws WalMarkerException { - updateState(tsi, DfsWalProvider.PROVIDER_ID, path, state); - } - private void updateState(TServerInstance tsi, String providerId, Path path, WalState state) throws WalMarkerException { byte[] data = (state + "," + providerId + "," + path).getBytes(UTF_8); @@ -162,11 +152,6 @@ private void updateState(TServerInstance tsi, String providerId, Path path, WalS } } - // Tablet server has no references to the WAL - public void walUnreferenced(TServerInstance tsi, Path path) throws WalMarkerException { - updateState(tsi, path, WalState.UNREFERENCED); - } - public void walUnreferenced(TServerInstance tsi, LogEntry entry) throws WalMarkerException { updateState(tsi, entry.getProviderId(), entry.getPath(), WalState.UNREFERENCED); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java index 4b7b4c83db7..65a46f09496 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java @@ -46,7 +46,6 @@ import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.LogSortInformation; import org.apache.accumulo.server.log.SortedLogState; -import org.apache.accumulo.server.manager.recovery.HadoopLogCloser; import org.apache.accumulo.server.manager.recovery.LogCloser; import org.apache.accumulo.server.manager.recovery.RecoveryPath; import org.apache.accumulo.server.zookeeper.DistributedWorkQueue; @@ -229,10 +228,9 @@ private boolean recoverLog(LogEntry walog) throws IOException { synchronized (this) { if (!closeTasksQueued.contains(sortIdString) && !sortsQueued.contains(sortIdString)) { AccumuloConfiguration aconf = manager.getConfiguration(); - LogCloser closer = walog.getProviderId().equals(DfsWalProvider.PROVIDER_ID) - ? Property.createInstanceFromPropertyName(aconf, - Property.MANAGER_WAL_CLOSER_IMPLEMENTATION, LogCloser.class, new HadoopLogCloser()) - : null; + // TODO: Should the LogCloser implementation be a part of the WalProvider SPI? + LogCloser closer = Property.createInstanceFromPropertyName(aconf, + Property.MANAGER_WAL_CLOSER_IMPLEMENTATION, LogCloser.class, null); Long delay = recoveryDelay.get(sortIdString); if (delay == null) { delay = aconf.getTimeInMillis(Property.MANAGER_RECOVERY_DELAY); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java index 6291dd3b3d0..0677c0dc2df 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java @@ -233,7 +233,7 @@ public static String getRowPrefix() { @VisibleForTesting static final Set UPGRADE_FAMILIES = Set.of(DataFileColumnFamily.NAME, CHOPPED, - ExternalCompactionColumnFamily.NAME, ScanFileColumnFamily.NAME, LogColumnFamily.NAME); + ExternalCompactionColumnFamily.NAME, LogColumnFamily.NAME, ScanFileColumnFamily.NAME); @VisibleForTesting static final String ZNAMESPACE_NAME = "/name"; From cfdc5db79c8580a327121d30826107c88de9aca9 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Fri, 13 Nov 2026 21:53:29 +0000 Subject: [PATCH 7/8] Moved the LogCloser into the WalProvider SPI --- WAL-SPI.md | 53 +++++++++++--- .../apache/accumulo/core/conf/Property.java | 3 - .../accumulo/core/spi/wal/WalProvider.java | 13 ++++ .../apache/accumulo/server/fs/VolumeUtil.java | 3 - .../accumulo/server/log/DfsWalProvider.java | 41 ++++++++--- .../recovery => log}/HadoopLogCloser.java | 2 +- .../{manager/recovery => log}/LogCloser.java | 7 +- .../server/log/LogSortInformation.java | 9 +++ .../recovery => log}/NoOpLogCloser.java | 7 +- .../server/log/LogSortInformationTest.java | 7 ++ .../manager/recovery/RecoveryManager.java | 28 ++++---- .../manager/upgrade/Upgrader11to12.java | 47 +++++++++++-- .../manager/upgrade/Upgrader11to12Test.java | 69 +++++++++++++++++++ .../accumulo/tserver/log/DfsLogger.java | 4 +- .../accumulo/tserver/log/LogSorter.java | 23 ++----- .../tserver/log/TestUpgradePathForWALogs.java | 6 +- shell/pom.xml | 5 -- .../commands/ListTabletsCommandTest.java | 5 +- .../test/wal/InMemoryWalProvider.java | 7 ++ 19 files changed, 257 insertions(+), 82 deletions(-) rename server/base/src/main/java/org/apache/accumulo/server/{manager/recovery => log}/HadoopLogCloser.java (98%) rename server/base/src/main/java/org/apache/accumulo/server/{manager/recovery => log}/LogCloser.java (85%) rename server/base/src/main/java/org/apache/accumulo/server/{manager/recovery => log}/NoOpLogCloser.java (91%) diff --git a/WAL-SPI.md b/WAL-SPI.md index feeb4447c33..71d64edd9f0 100644 --- a/WAL-SPI.md +++ b/WAL-SPI.md @@ -21,8 +21,8 @@ # Write-ahead log storage SPI (current implementation) Accumulo's WAL storage SPI lets a TabletServer create raw write-ahead logs using a configured -`WalProvider`. The provider owns storage of the raw byte stream; Accumulo owns the WAL format, -encryption, tablet bookkeeping, sorting, and replay. The implementation is split between the +`WalProvider`. The provider owns storage of the raw byte stream and preparation for recovery; +Accumulo owns the WAL format, encryption, tablet bookkeeping, sorting, and replay. The implementation is split between the [SPI interface][spi], the [provider registry][registry], and the built-in [filesystem provider][dfs-provider]. @@ -49,6 +49,18 @@ For a provider with options, put string-valued options in its map, for example [distribution configuration][dist-config] shows these settings; their definitions are in [Property.java][property]. +`DfsWalProvider` accepts an optional `logCloser` option naming a public no-argument implementation +of `org.apache.accumulo.server.manager.recovery.LogCloser`. It defaults to +`org.apache.accumulo.server.manager.recovery.HadoopLogCloser`. For a filesystem that needs no lease +recovery, configure the existing `NoOpLogCloser`: + +```properties +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{"logCloser":"org.apache.accumulo.server.manager.recovery.NoOpLogCloser"}} +``` + +Closer implementations must support repeated calls and concurrent preparation of different WALs. +Provider options are site configuration; changes require restarting the affected server processes. + Each implementation needs a public no-argument constructor and a stable `getId()` matching `[A-Za-z0-9]+`. The ID `dfs` is reserved for `DfsWalProvider` and legacy filesystem WALs. The [registry][registry] loads each configured class, passes its options to `init(InitParameters)` once per @@ -73,6 +85,9 @@ WALs without moving or rewriting the WAL files: - Legacy ZooKeeper WAL marker data `,` becomes `,dfs,`. Markers already containing a provider ID are retained. The upgrader checks marker format and verifies that an updated marker was persisted. +- Legacy recovery work `source|destination` becomes `sortId|dfs|source|destination`, using the + recovery node's UUID as `sortId`. Converted work is validated and retained on retries; the `locks` + node is skipped. Invalid provider IDs, blank paths, mismatched IDs, or failed writes stop the upgrade. - The upgrader checks for remaining legacy `-/` log qualifiers in the metadata tables; it also verifies the root tablet's WAL references. Malformed markers or leftover legacy qualifiers cause the upgrade to fail rather than silently treating them as another provider's logs. @@ -83,6 +98,13 @@ for new writes. Otherwise the Manager's provider-reference check or subsequent r cannot handle those existing `dfs` references. The upgrade changes **reference metadata**, not the raw WAL format or the sorted-log recovery process. +The global `manager.wal.closer.implementation` property is removed in 4.0. Before upgrading, +move custom site-file settings into the DFS provider's `logCloser` option and remove the old +site-file property. The upgrader removes an explicitly stored ZooKeeper setting after verifying +that any non-default closer has the same replacement configured. The default Hadoop closer needs +no explicit provider option. An unmigrated setting stops the upgrade rather than silently changing +recovery behavior. + ## Provider contract [`WalProvider`][spi] exposes: @@ -90,14 +112,21 @@ raw WAL format or the sorted-log recovery process. | Method | Use | | ----------------------------------- | ----------------------------------------------------- | | `create(serverAddress, UUID)` | Exclusively create a WAL and return its `Writer`. | +| `prepareForRecovery(reference)` | Make an abandoned WAL consistently readable for sorting. | | `open(reference)` | Read a WAL from byte zero for sorting/recovery. | | `exists(reference)` | Check a WAL, including while it may still be written. | | `delete(reference)` | Remove an unreferenced WAL; should be idempotent. | | `getId()` / `init(InitParameters)` | Supply the persisted ID and initialize the provider. | +Recovery preparation is required for every provider. It performs one storage-level lease-recovery, +finalization, or fencing attempt and returns zero when ready, or a positive retry delay in +milliseconds. Negative delays are invalid. Calls must be safe to repeat after retries or Manager +failover and may run concurrently for different WALs. A provider needing no preparation returns zero. + The nested `Writer` supplies `getReference()`, `getOutputStream()`, `sync(Durability)`, -`getLength()`, and `close()`. It can optionally report `getCurrentReplication()` and -`getPipeline()` for diagnostics (the defaults return `-1` and an empty array). Accumulo writes the +`getLength()`, `isHealthy()`, `getDiagnostics()`, and `close()`. Health and diagnostic methods are +required: providers without additional health information can return true and an empty diagnostic +string. Accumulo checks health after slow syncs and fails pending writes when unhealthy. Accumulo writes the WAL header and records to the output stream: a provider must retain the exact bytes in order and return them through `open()`. Providers must be thread-safe for operations on different logs; a writer must also coordinate concurrent appends and sync requests. `FLUSH` must survive a @@ -121,8 +150,8 @@ corresponding recovery destination are current integration constraints. header and crypto parameters, wraps the provider output stream for encryption, writes an `OPEN` entry, and syncs it before using the log. Subsequent tablet definitions, mutations, and compaction events use the existing WAL serialization. A background sync task batches durability - requests and calls `Writer.sync(FLUSH)` or `Writer.sync(SYNC)`; it uses the optional - replication/pipeline methods for diagnostics. + requests and calls `Writer.sync(FLUSH)` or `Writer.sync(SYNC)`; after slow syncs it calls + `getDiagnostics()` and `isHealthy()`. 3. `LogEntry` associates the provider ID with the path. Tablet metadata stores it as a log-column qualifier of the form `/`. [`WalStateManager`][wal-state] records the WAL state (`OPEN`, `CLOSED`, or `UNREFERENCED`), provider ID, and path in ZooKeeper WAL markers. This @@ -133,14 +162,18 @@ corresponding recovery destination are current integration constraints. creates a filesystem-backed WAL under `wal//` using `VolumeManager`; its writer uses `hflush()` for `FLUSH` and `hsync()` for `SYNC`. It opens and checks files through `VolumeManager`, and uses the filesystem trash or recursive deletion for cleanup. Where the -underlying stream is an HDFS `DFSOutputStream`, it can report replication and pipeline information. +underlying stream is a replicated HDFS `DFSOutputStream`, its writer records the observed replication +after a successful sync and reports unhealthy if replication falls below that baseline. Local and +striped streams are exempt from that replicated-stream check. Pipeline diagnostics tolerate an +unavailable pipeline. ## Recovery and garbage collection The [Manager's `RecoveryManager`][rm] obtains the provider ID from the `LogEntry`, -derives the sibling `recovery/` directory using `RecoveryPath`, and queues a sort. Filesystem -WALs with ID `dfs` use the configured Hadoop log closer before sorting; other providers do not use -that closer. The queued work carries the provider ID, source reference, and recovery destination. +derives the sibling `recovery/` directory using `RecoveryPath`, and calls that provider's +`prepareForRecovery(reference)`. The Manager schedules requested retries and queues sorting only +when preparation returns zero. `DfsWalProvider` delegates preparation to its configured log closer. +The queued work is serialized as `sortId|providerId|source|destination`. [`LogSorter`][sorter] opens the raw source through `VolumeManager` for `dfs` or through the registered provider's `open(reference)` for another ID. It decrypts and decodes Accumulo's WAL diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 33841e3d600..44a02d48e83 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -503,9 +503,6 @@ are performed (e.g. Bulk Import). This property specifies the maximum number of MANAGER_LEASE_RECOVERY_WAITING_PERIOD("manager.lease.recovery.interval", "5s", PropertyType.TIMEDURATION, "The amount of time to wait after requesting a write-ahead log to be recovered.", "1.5.0"), - MANAGER_WAL_CLOSER_IMPLEMENTATION("manager.wal.closer.implementation", - "org.apache.accumulo.server.manager.recovery.HadoopLogCloser", PropertyType.CLASSNAME, - "A class that implements a mechanism to steal write access to a write-ahead log.", "2.1.0"), MANAGER_FATE_CONDITIONAL_WRITER_THREADS_MAX("manager.fate.conditional.writer.threads.max", "3", PropertyType.COUNT, "Maximum number of threads to use for writing data to tablet servers of the FATE system table.", diff --git a/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java index fe86d9f003c..7bdf5a8e591 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java @@ -79,6 +79,19 @@ interface InitParameters { Map getOptions(); } + /** + * Prepares a closed or abandoned WAL for recovery, including any storage-level lease recovery or + * fencing needed to make its recoverable bytes consistently readable through + * {@link #open(String)}. Accumulo calls this after the TabletServer has closed or lost ownership + * of the WAL. Calls must be safe to repeat after retries or Manager failover, and may run + * concurrently for different WALs. Perform one attempt rather than waiting indefinitely; Accumulo + * schedules retries. + * + * @param reference reference returned by {@link Writer#getReference()} + * @return zero when ready, or a positive delay in milliseconds before retrying; never negative + */ + long prepareForRecovery(String reference) throws IOException; + /** * Creates a new exclusive WAL for the given TabletServer and UUID. The returned reference must be * unique, fully qualified, and end in the supplied server address (with {@code :} replaced by diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java index 63c8db1477a..ac61c4ca06e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java @@ -81,9 +81,6 @@ public static Path switchVolume(Path path, FileType ft, Map replaceme } public static LogEntry switchVolume(LogEntry le, Map replacements) { - if (!le.getProviderId().equals(DfsWalProvider.PROVIDER_ID)) { - return null; - } Path switchedPath = switchVolume(le.getPath(), FileType.WAL, replacements); return switchedPath == null ? null : LogEntry.withSwitchedVolume(le, switchedPath); } diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java index c7f514d49fc..c32e952f4df 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java @@ -47,7 +47,11 @@ public class DfsWalProvider implements WalProvider { /** Stable provider ID used for WALs stored on HDFS. */ public static final String PROVIDER_ID = "dfs"; + /** Provider option selecting a public no-argument {@link LogCloser} implementation. */ + public static final String LOG_CLOSER_OPTION = "logCloser"; + private ServerContext context; + private LogCloser logCloser; @Override public String getId() { @@ -61,6 +65,19 @@ public void init(InitParameters params) { throw new IllegalArgumentException("DfsWalProvider requires the server service environment"); } context = serviceEnvironment.getContext(); + String className = + params.getOptions().getOrDefault(LOG_CLOSER_OPTION, HadoopLogCloser.class.getName()); + try { + logCloser = environment.instantiate(className, LogCloser.class); + } catch (ReflectiveOperationException | RuntimeException e) { + throw new IllegalArgumentException("Unable to load DFS WAL log closer " + className, e); + } + } + + @Override + public long prepareForRecovery(String reference) throws IOException { + return logCloser.close(context.getConfiguration(), context.getHadoopConf(), + context.getVolumeManager(), new Path(reference)); } @Override @@ -85,6 +102,8 @@ public Writer create(String serverAddress, UUID id) throws IOException { if (context.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC)) { output = fs.createSyncable(path, 0, replication, blockSize); } else { + // TODO: 2nd parameter here is `true`, which signifies to overwrite + // an existing file. Is that correct? output = fs.create(path, true, 0, replication, blockSize); } try { @@ -123,6 +142,7 @@ private static class DfsWriter implements Writer { private final String reference; private final FSDataOutputStream output; private boolean closed; + private int expectedReplication; DfsWriter(String reference, FSDataOutputStream output) { this.reference = reference; @@ -148,6 +168,11 @@ public void sync(Durability durability) throws IOException { } else { throw new IllegalArgumentException("Unexpected WAL sync durability " + durability); } + var wrapped = output.getWrappedStream(); + if (expectedReplication == 0 && wrapped instanceof DFSOutputStream dfs + && !(wrapped instanceof DFSStripedOutputStream)) { + expectedReplication = dfs.getCurrentBlockReplication(); + } } @Override @@ -158,13 +183,11 @@ public long getLength() throws IOException { @Override public boolean isHealthy() throws IOException { var wrapped = output.getWrappedStream(); - if (wrapped instanceof DFSOutputStream repl) { - return repl.getCurrentBlockReplication() > 1; - } else if (wrapped instanceof DFSStripedOutputStream) { - return true; - } else { - return false; + if (expectedReplication > 0 && wrapped instanceof DFSOutputStream dfs + && !(wrapped instanceof DFSStripedOutputStream)) { + return dfs.getCurrentBlockReplication() >= expectedReplication; } + return true; } @Override @@ -173,8 +196,10 @@ public String getDiagnostics() throws IOException { var wrapped = output.getWrappedStream(); if (wrapped instanceof DFSOutputStream dfs) { DatanodeInfo[] pipeline = dfs.getPipeline(); - for (int i = 0; i < pipeline.length; i++) { - buf.append("name: ").append(pipeline[i].getName()).append("\n"); + if (pipeline != null) { + for (DatanodeInfo node : pipeline) { + buf.append("name: ").append(node.getName()).append('\n'); + } } } return buf.toString(); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/recovery/HadoopLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/log/HadoopLogCloser.java similarity index 98% rename from server/base/src/main/java/org/apache/accumulo/server/manager/recovery/HadoopLogCloser.java rename to server/base/src/main/java/org/apache/accumulo/server/log/HadoopLogCloser.java index 2c0409b80c5..640ff129ef8 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/recovery/HadoopLogCloser.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/HadoopLogCloser.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.server.manager.recovery; +package org.apache.accumulo.server.log; import java.io.FileNotFoundException; import java.io.IOException; diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/recovery/LogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/log/LogCloser.java similarity index 85% rename from server/base/src/main/java/org/apache/accumulo/server/manager/recovery/LogCloser.java rename to server/base/src/main/java/org/apache/accumulo/server/log/LogCloser.java index f4013035a62..e1a2211d9f3 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/recovery/LogCloser.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/LogCloser.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.server.manager.recovery; +package org.apache.accumulo.server.log; import java.io.IOException; @@ -27,7 +27,10 @@ /** * Object that is used by the RecoveryManager to properly close WALogs that were being written to - * and were not properly closed (TabletServer died, for example). + * and were not properly closed (TabletServer died, for example). Configured implementations must + * have a public no-argument constructor and support concurrent calls for different WALs. A DFS WAL + * provider reuses its closer, and recovery attempts may be repeated after retries or Manager + * failover. */ public interface LogCloser { diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/LogSortInformation.java b/server/base/src/main/java/org/apache/accumulo/server/log/LogSortInformation.java index 2cddb3d706f..b08fdaadcb1 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/LogSortInformation.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/LogSortInformation.java @@ -18,10 +18,13 @@ */ package org.apache.accumulo.server.log; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import java.util.UUID; +import org.apache.accumulo.core.spi.wal.WalProvider; + /** * The information needed to sort a write-ahead log, and its representation in the recovery work * queue. @@ -34,6 +37,12 @@ public record LogSortInformation(UUID sortId, String providerId, String source, requireNonNull(providerId, "providerId is null"); requireNonNull(source, "source is null"); requireNonNull(destination, "destination is null"); + checkArgument(providerId.matches(WalProvider.PROVIDER_ID_REGEX), "Invalid WAL provider ID: %s", + providerId); + checkArgument(!source.isBlank() && !destination.isBlank(), + "Recovery source and destination must not be blank"); + checkArgument(source.indexOf('|') < 0 && destination.indexOf('|') < 0, + "Recovery source and destination must not contain '|'"); } /** Serialize this work item for storage in ZooKeeper. */ diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/recovery/NoOpLogCloser.java b/server/base/src/main/java/org/apache/accumulo/server/log/NoOpLogCloser.java similarity index 91% rename from server/base/src/main/java/org/apache/accumulo/server/manager/recovery/NoOpLogCloser.java rename to server/base/src/main/java/org/apache/accumulo/server/log/NoOpLogCloser.java index d5665900027..34b7cb76b8e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/recovery/NoOpLogCloser.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/NoOpLogCloser.java @@ -16,12 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.server.manager.recovery; +package org.apache.accumulo.server.log; import java.io.IOException; import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -39,8 +38,8 @@ * recovery will not work if the file system implementation needs some type of recovery to occur for * files that were being written to, but not properly closed. * - * To use this class, set the Property {@link Property#MANAGER_WAL_CLOSER_IMPLEMENTATION} to the - * full name of this class. + * + * To use this class, set the {@link DfsWalProvider#LOG_CLOSER_OPTION} option for + * + * {@link DfsWalProvider} in {@code instance.wal.providers} to the full name of this class. */ public class NoOpLogCloser implements LogCloser { diff --git a/server/base/src/test/java/org/apache/accumulo/server/log/LogSortInformationTest.java b/server/base/src/test/java/org/apache/accumulo/server/log/LogSortInformationTest.java index 3b3e9bc7b88..5045971c2d5 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/log/LogSortInformationTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/log/LogSortInformationTest.java @@ -38,6 +38,13 @@ void roundTrip() { @Test void malformedPayload() { + String id = UUID.randomUUID().toString(); + for (String payload : new String[] {id + "||source|dest", id + "|bad-id|source|dest", + id + "|dfs||dest", id + "|dfs|source| "}) { + assertThrows(IllegalArgumentException.class, () -> LogSortInformation.deserialize(payload)); + } + assertThrows(IllegalArgumentException.class, + () -> new LogSortInformation(UUID.randomUUID(), "dfs", "source|extra", "dest")); assertThrows(IllegalArgumentException.class, () -> LogSortInformation.deserialize("source|dest")); assertThrows(IllegalArgumentException.class, diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java index 65a46f09496..a8cd59d9efc 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java @@ -38,15 +38,14 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.spi.wal.WalProvider; import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.cache.Caches.CacheName; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.manager.Manager; import org.apache.accumulo.server.fs.VolumeUtil; -import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.server.log.LogSortInformation; import org.apache.accumulo.server.log.SortedLogState; -import org.apache.accumulo.server.manager.recovery.LogCloser; import org.apache.accumulo.server.manager.recovery.RecoveryPath; import org.apache.accumulo.server.zookeeper.DistributedWorkQueue; import org.apache.hadoop.fs.Path; @@ -91,11 +90,11 @@ private class LogSortTask implements Runnable { private final String destination; private final UUID sortId; private final String providerId; - private final LogCloser closer; + private final WalProvider provider; - public LogSortTask(LogCloser closer, String providerId, String source, String destination, + public LogSortTask(WalProvider provider, String providerId, String source, String destination, UUID sortId) { - this.closer = closer; + this.provider = provider; this.providerId = providerId; this.source = source; this.destination = destination; @@ -106,8 +105,10 @@ public LogSortTask(LogCloser closer, String providerId, String source, String de public void run() { boolean rescheduled = false; try { - long time = closer == null ? 0 : closer.close(manager.getConfiguration(), - manager.getContext().getHadoopConf(), manager.getVolumeManager(), new Path(source)); + long time = provider.prepareForRecovery(source); + if (time < 0) { + throw new IllegalArgumentException("Negative recovery retry delay from " + providerId); + } if (time > 0) { ScheduledFuture future = executor.schedule(this, time, TimeUnit.MILLISECONDS); @@ -186,8 +187,8 @@ public RecoverySession newRecoverySession() { private boolean recoverLog(LogEntry walog) throws IOException { boolean recoveryNeeded = false; - LogEntry switchedWalog = walog.getProviderId().equals(DfsWalProvider.PROVIDER_ID) - ? VolumeUtil.switchVolume(walog, manager.getContext().getVolumeReplacements()) : null; + LogEntry switchedWalog = + VolumeUtil.switchVolume(walog, manager.getContext().getVolumeReplacements()); if (switchedWalog != null) { // replaces the volume used for sorting, but do not change entry in metadata table. When // the tablet loads it will change the metadata table entry. If @@ -199,7 +200,7 @@ private boolean recoverLog(LogEntry walog) throws IOException { UUID sortId = walog.getUniqueID(); String sortIdString = sortId.toString(); String filename = walog.getPath().toString(); - String dest = RecoveryPath.getRecoveryPath(new Path(filename)).toString(); + String dest = RecoveryPath.getRecoveryPath(walog.getPath()).toString(); boolean sortQueued; synchronized (this) { @@ -228,9 +229,8 @@ private boolean recoverLog(LogEntry walog) throws IOException { synchronized (this) { if (!closeTasksQueued.contains(sortIdString) && !sortsQueued.contains(sortIdString)) { AccumuloConfiguration aconf = manager.getConfiguration(); - // TODO: Should the LogCloser implementation be a part of the WalProvider SPI? - LogCloser closer = Property.createInstanceFromPropertyName(aconf, - Property.MANAGER_WAL_CLOSER_IMPLEMENTATION, LogCloser.class, null); + WalProvider provider = + manager.getContext().getWalProviderRegistry().get(walog.getProviderId()); Long delay = recoveryDelay.get(sortIdString); if (delay == null) { delay = aconf.getTimeInMillis(Property.MANAGER_RECOVERY_DELAY); @@ -241,7 +241,7 @@ private boolean recoverLog(LogEntry walog) throws IOException { log.info("Starting recovery of {} (in : {}s)", filename, (delay / 1000)); ScheduledFuture future = executor.schedule( - new LogSortTask(closer, walog.getProviderId(), filename, dest, sortId), delay, + new LogSortTask(provider, walog.getProviderId(), filename, dest, sortId), delay, TimeUnit.MILLISECONDS); ThreadPools.watchNonCriticalScheduledTask(future); closeTasksQueued.add(sortIdString); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java index 0677c0dc2df..5aa5bd97f53 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java @@ -106,6 +106,7 @@ import org.apache.accumulo.server.init.FileSystemInitializer; import org.apache.accumulo.server.init.InitialConfiguration; import org.apache.accumulo.server.log.DfsWalProvider; +import org.apache.accumulo.server.log.HadoopLogCloser; import org.apache.accumulo.server.log.LogSortInformation; import org.apache.accumulo.server.log.WalStateManager; import org.apache.accumulo.server.util.FindCompactionTmpFiles.DeleteStats; @@ -121,6 +122,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.gson.JsonParser; //TODO when removing this class, also remove MetadataSchema.Upgrader11to12 public class Upgrader11to12 implements Upgrader { @@ -241,10 +243,14 @@ public static String getRowPrefix() { public static final Collection OLD_SCAN_SERVERS_RANGES = List.of(new Range("~sserv", "~sserx"), new Range("~scanref", "~scanreg")); + private static final String LEGACY_LOG_CLOSER_PROPERTY = "manager.wal.closer.implementation"; + @Override public void upgradeZookeeper(ServerContext context) { LOG.info("Ensuring all worker server processes are down."); validateEmptyZKWorkerServerPaths(context); + LOG.info("Migrating WAL closer configuration"); + upgradeWalCloserConfiguration(context); LOG.info("Upgrading WAL provider IDs in ZooKeeper markers"); upgradeWalMarkers(context); LOG.info("Updating recovery work payloads"); @@ -283,6 +289,36 @@ public void upgradeZookeeper(ServerContext context) { addShuttingDownTservers(context); } + @VisibleForTesting + void upgradeWalCloserConfiguration(ServerContext context) { + if (context.getSiteConfiguration().get(LEGACY_LOG_CLOSER_PROPERTY) != null) { + throw new IllegalStateException("Remove " + LEGACY_LOG_CLOSER_PROPERTY + + " from site configuration and configure the DFS WAL provider's logCloser option instead"); + } + var key = SystemPropKey.of(); + String oldCloser = context.getPropStore().get(key).asMap().get(LEGACY_LOG_CLOSER_PROPERTY); + if (oldCloser != null) { + validateWalCloserConfiguration(oldCloser, + context.getConfiguration().get(Property.INSTANCE_WAL_PROVIDERS)); + context.getPropStore().removeProperties(key, Set.of(LEGACY_LOG_CLOSER_PROPERTY)); + } + } + + static void validateWalCloserConfiguration(String oldCloser, String providers) { + if (oldCloser.equals(HadoopLogCloser.class.getName())) { + return; + } + var options = JsonParser.parseString(providers).getAsJsonObject() + .getAsJsonObject(DfsWalProvider.class.getName()); + if (options == null || !options.has(DfsWalProvider.LOG_CLOSER_OPTION) + || !oldCloser.equals(options.get(DfsWalProvider.LOG_CLOSER_OPTION).getAsString())) { + throw new IllegalStateException("Configure " + DfsWalProvider.class.getName() + " option " + + DfsWalProvider.LOG_CLOSER_OPTION + " as " + oldCloser + " in " + + Property.INSTANCE_WAL_PROVIDERS.getKey() + " before removing " + + LEGACY_LOG_CLOSER_PROPERTY); + } + } + private void updateRecoveryWork(ServerContext context) { try { upgradeRecoveryWork(context.getZooSession().asReaderWriter()); @@ -490,10 +526,6 @@ private void removeCompactColumnsFromRootTabletMetadata(ServerContext context) { zrw.overwritePersistentData(ZROOT_TABLET, rtm.toJson().getBytes(UTF_8), stat.getVersion()); LOG.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson()); } - RootTabletMetadata upgraded = - new RootTabletMetadata(new String(zrw.getData(ZROOT_TABLET), UTF_8)); - upgraded.getKeyValues() - .forEach(entry -> verifyNoLegacyWalReference(entry.getKey(), "root tablet metadata")); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); throw new IllegalStateException( @@ -1104,10 +1136,13 @@ void updateRootTabletFileReferences(ServerContext context) { if (!mutations.isEmpty()) { LOG.info("Root metadata in ZooKeeper before upgrade: {}", json); rtm.update(mutations.get(0)); - zrw.overwritePersistentData(ZROOT_TABLET, rtm.toJson().getBytes(UTF_8), stat.getVersion()); + if (!zrw.overwritePersistentData(ZROOT_TABLET, rtm.toJson().getBytes(UTF_8), + stat.getVersion())) { + throw new IllegalStateException("Could not update root tablet file references"); + } LOG.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson()); } - rtm.getKeyValues() + new RootTabletMetadata(new String(zrw.getData(ZROOT_TABLET), UTF_8)).getKeyValues() .forEach(entry -> verifyNoLegacyWalReference(entry.getKey(), "root tablet")); } catch (KeeperException | InterruptedException e) { if (e instanceof InterruptedException) { diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java index 7a2b9985264..87fec9cdd1f 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java @@ -87,7 +87,9 @@ import org.apache.accumulo.server.conf.store.impl.ZooPropStore; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.log.DfsWalProvider; +import org.apache.accumulo.server.log.HadoopLogCloser; import org.apache.accumulo.server.log.LogSortInformation; +import org.apache.accumulo.server.log.NoOpLogCloser; import org.apache.accumulo.server.log.WalStateManager; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -577,6 +579,7 @@ public void upgradeZooKeeperTest() throws Exception { Capture byteCapture = newCapture(); expect(zrw.overwritePersistentData(eq("/root_tablet"), capture(byteCapture), eq(123))) .andReturn(true).once(); + expect(zrw.getData("/root_tablet")).andAnswer(byteCapture::getValue); expect(zrw.getData(eq(Constants.ZNAMESPACES))).andReturn(new byte[0]).once(); Map mockNamespaces = Map.of("ns1", "ns1name", "ns2", "ns2name"); @@ -882,4 +885,70 @@ public void testMoveTableProperties() throws Exception { verify(context, propStore, sysVerProps, systemNsProps, defaultNsProps, testNsProps, nsmap); } + + @Test + void testMalformedRecoveryWork() { + String id = UUID.randomUUID().toString(); + for (String payload : List.of("source", "|destination", "source|", id + "||source|destination", + id + "|bad-id|source|destination", id + "|dfs||destination")) { + assertThrows(IllegalArgumentException.class, + () -> Upgrader11to12.convertRecoveryWork(id, payload), payload); + } + } + + @Test + void testRecoveryWorkWriteFailure() throws Exception { + ZooReaderWriter zrw = createStrictMock(ZooReaderWriter.class); + String id = UUID.randomUUID().toString(); + String path = Constants.ZRECOVERY + "/" + id; + expect(zrw.getChildren(Constants.ZRECOVERY)).andReturn(List.of(id)); + expect(zrw.getData(path)).andReturn("source|destination".getBytes(UTF_8)); + expect(zrw.overwritePersistentData(eq(path), + aryEq((id + "|dfs|source|destination").getBytes(UTF_8)), eq(-1))).andReturn(false); + replay(zrw); + assertThrows(IllegalStateException.class, () -> new Upgrader11to12().upgradeRecoveryWork(zrw)); + verify(zrw); + } + + @Test + void testWalCloserConfigurationMigration() { + String dfs = DfsWalProvider.class.getName(); + String custom = NoOpLogCloser.class.getName(); + Upgrader11to12.validateWalCloserConfiguration(HadoopLogCloser.class.getName(), + "{\"" + dfs + "\":{}}"); + Upgrader11to12.validateWalCloserConfiguration(custom, + "{\"" + dfs + "\":{\"logCloser\":\"" + custom + "\"}}"); + assertThrows(IllegalStateException.class, + () -> Upgrader11to12.validateWalCloserConfiguration(custom, "{\"" + dfs + "\":{}}")); + } + + @Test + void testRootWalUpgradePersistsAndRetries() throws Exception { + String reference = "hdfs://nn/accumulo/wal/host+1234/" + UUID.randomUUID(); + String legacy = "{\"version\":1,\"columnValues\":{\"log\":{\"-/" + reference + "\":\"\"}}}"; + ServerContext context = createMock(ServerContext.class); + VolumeManager volumes = createMock(VolumeManager.class); + ZooSession session = createMock(ZooSession.class); + ZooReaderWriter zoo = createStrictMock(ZooReaderWriter.class); + Capture persisted = newCapture(); + expect(context.getZooSession()).andReturn(session).times(2); + expect(session.asReaderWriter()).andReturn(zoo).times(2); + expect(context.getVolumeManager()).andReturn(volumes).times(2); + expect(volumes.getVolumes()).andReturn(List.of()).times(2); + expect(zoo.getData(eq("/root_tablet"), isA(Stat.class))).andReturn(legacy.getBytes(UTF_8)); + expect(zoo.overwritePersistentData(eq("/root_tablet"), capture(persisted), eq(0))) + .andReturn(true); + expect(zoo.getData("/root_tablet")).andAnswer(persisted::getValue); + expect(zoo.getData(eq("/root_tablet"), isA(Stat.class))).andAnswer(persisted::getValue); + expect(zoo.getData("/root_tablet")).andAnswer(persisted::getValue); + replay(context, volumes, session, zoo); + Upgrader11to12 upgrader = new Upgrader11to12(); + upgrader.updateRootTabletFileReferences(context); + upgrader.updateRootTabletFileReferences(context); + var entries = new RootTabletMetadata(new String(persisted.getValue(), UTF_8)).toKeyValues(); + assertEquals(1, entries.size()); + assertEquals("dfs/" + reference, entries.firstKey().getColumnQualifier().toString()); + verify(context, volumes, session, zoo); + } + } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java index 4e8034a487f..0f5df41fea0 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java @@ -185,8 +185,8 @@ public void run() { log.info("Slow sync cost: {} ms, current information: {}", timer.elapsed(MILLISECONDS), logFile.getDiagnostics()); if (!logFile.isHealthy()) { - fail(work, new IOException("WalProvider is reporting it's not healthly"), - "replication check"); + fail(work, new IOException("WAL provider reports an unhealthy writer"), + "writer health check"); } } } catch (IOException | RuntimeException ex) { diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java index 79c091652d3..7a328ff6d9b 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java @@ -159,11 +159,6 @@ public void process(String child, byte[] data) { } } - public void sort(VolumeManager fs, String name, Path srcPath, String destPath) - throws IOException { - sort(fs, name, DfsWalProvider.PROVIDER_ID, srcPath, destPath); - } - public void sort(VolumeManager fs, String name, String providerId, Path srcPath, String destPath) throws IOException { int part = 0; @@ -244,7 +239,7 @@ public synchronized long getSortTime() { return 0; } - synchronized long getBytesCopied() throws IOException { + synchronized long getBytesCopied() { return input == null ? bytesCopied : input.getCount(); } } @@ -354,13 +349,9 @@ public List getLogSorts() { for (Entry entries : currentWork.entrySet()) { RecoveryStatus status = new RecoveryStatus(); status.setName(entries.getKey()); - try { - double progress = entries.getValue().getBytesCopied() / walBlockSize; - // to be sure progress does not exceed 100% - status.setProgress(Math.min(progress, 99.9)); - } catch (IOException ex) { - log.warn("Error getting bytes read"); - } + double progress = entries.getValue().getBytesCopied() / walBlockSize; + // to be sure progress does not exceed 100% + status.setProgress(Math.min(progress, 99.9)); status.setRuntime((int) entries.getValue().getSortTime()); result.add(status); } @@ -382,11 +373,7 @@ private void updateMetrics() { if (start > 0) { // may not have started yet runtime = Math.max(start, runtime); } - try { - progress += processor.getBytesCopied(); - } catch (IOException e) { - log.warn("Error getting bytes read"); - } + progress += processor.getBytesCopied(); } recoveryRuntime.set(runtime); recoveryAvgProgress diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java index 0e5c74032a5..1f5b561cd7a 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java @@ -116,7 +116,7 @@ public void testUpgradeOf15WALog() throws IOException { LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor(); assertThrows(IllegalArgumentException.class, - () -> logProcessor.sort(context.getVolumeManager(), "/" + WALOG_FROM_15, + () -> logProcessor.sort(context.getVolumeManager(), "/" + WALOG_FROM_15, "dfs", new Path(testPath.toUri() + WALOG_FROM_15), testPath.toUri() + "manyMaps")); } @@ -132,7 +132,7 @@ public void testBasic16WALogRead() throws IOException { LogSorter logSorter = new LogSorter(server); LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor(); - logProcessor.sort(context.getVolumeManager(), "/" + walogToTest, + logProcessor.sort(context.getVolumeManager(), "/" + walogToTest, "dfs", new Path(testPath.toUri() + walogToTest), destPath); assertTrue(context.getVolumeManager().exists(getFinishedMarkerPath(destPath))); @@ -149,7 +149,7 @@ public void testBasic20WALogRead() throws IOException { LogSorter logSorter = new LogSorter(server); LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor(); - logProcessor.sort(context.getVolumeManager(), "/" + walogToTest, + logProcessor.sort(context.getVolumeManager(), "/" + walogToTest, "dfs", new Path(testPath.toUri() + walogToTest), destPath); assertTrue(context.getVolumeManager().exists(getFinishedMarkerPath(destPath))); diff --git a/shell/pom.xml b/shell/pom.xml index 08294cf7a4d..b413fdcb4c3 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -91,11 +91,6 @@ hadoop-client-runtime runtime - - org.apache.accumulo - accumulo-server-base - test - org.apache.logging.log4j log4j-slf4j2-impl diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java index cb489ca3f71..861bb3b130a 100644 --- a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java +++ b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java @@ -48,7 +48,6 @@ import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.tabletserver.log.LogEntry; -import org.apache.accumulo.server.log.DfsWalProvider; import org.apache.accumulo.shell.Shell; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -192,9 +191,9 @@ public void mockTest() throws Exception { KeyExtent extent = new KeyExtent(tableId, new Text("d"), null); - LogEntry le1 = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, + LogEntry le1 = LogEntry.fromPath("exampleProvider", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); - LogEntry le2 = LogEntry.fromPath(DfsWalProvider.PROVIDER_ID, + LogEntry le2 = LogEntry.fromPath("exampleProvider", "file:/accumulo/wal/localhost+8020/" + UUID.randomUUID()); TabletMetadata tm1 = diff --git a/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java index d958d5c2c52..e04e642697b 100644 --- a/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java +++ b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java @@ -83,6 +83,13 @@ public void init(InitParameters params) { } } + @Override + public long prepareForRecovery(String reference) throws IOException { + requireInitialized(); + validateReference(reference); + return 0; + } + @Override public Writer create(String serverAddress, UUID id) throws IOException { requireInitialized(); From 7c6a971af83747da618dc372df73ae0479b0d358 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Sat, 14 Nov 2026 20:00:34 +0000 Subject: [PATCH 8/8] Removed tserver address from WalProvider.create, other minor changes --- WAL-SPI.md | 50 +++---- .../accumulo/core/spi/wal/WalProvider.java | 15 +- .../core/tabletserver/log/LogEntry.java | 134 +++++++++++------- .../core/tabletserver/log/LogEntryTest.java | 43 +++--- .../accumulo/server/log/DfsWalProvider.java | 8 +- .../accumulo/server/log/WalStateManager.java | 43 +++--- .../server/manager/recovery/RecoveryPath.java | 53 +++---- .../server/log/WalStateManagerTest.java | 7 +- .../manager/recovery/RecoveryPathTest.java | 55 +++++++ .../manager/recovery/RecoveryManager.java | 52 +++---- .../manager/upgrade/Upgrader11to12.java | 39 ++++- .../manager/upgrade/Upgrader11to12Test.java | 12 +- .../accumulo/tserver/log/DfsLogger.java | 20 ++- .../accumulo/tserver/log/LogSorter.java | 6 +- .../tserver/log/ResolvedSortedLog.java | 2 +- .../tserver/log/TabletServerLogger.java | 6 +- .../tserver/log/TestUpgradePathForWALogs.java | 12 +- .../org/apache/accumulo/test/RecoveryIT.java | 2 +- .../accumulo/test/upgrade/UpgradeUtilIT.java | 8 ++ .../test/wal/InMemoryWalProvider.java | 12 +- .../test/wal/InMemoryWalProviderTest.java | 11 +- 21 files changed, 349 insertions(+), 241 deletions(-) create mode 100644 server/base/src/test/java/org/apache/accumulo/server/manager/recovery/RecoveryPathTest.java diff --git a/WAL-SPI.md b/WAL-SPI.md index 71d64edd9f0..7b1dab4640f 100644 --- a/WAL-SPI.md +++ b/WAL-SPI.md @@ -50,12 +50,12 @@ For a provider with options, put string-valued options in its map, for example [Property.java][property]. `DfsWalProvider` accepts an optional `logCloser` option naming a public no-argument implementation -of `org.apache.accumulo.server.manager.recovery.LogCloser`. It defaults to -`org.apache.accumulo.server.manager.recovery.HadoopLogCloser`. For a filesystem that needs no lease -recovery, configure the existing `NoOpLogCloser`: +of `org.apache.accumulo.server.log.LogCloser`. It defaults to +`org.apache.accumulo.server.log.HadoopLogCloser`. For a filesystem that needs no lease recovery, +configure the existing `NoOpLogCloser`: ```properties -instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{"logCloser":"org.apache.accumulo.server.manager.recovery.NoOpLogCloser"}} +instance.wal.providers={"org.apache.accumulo.server.log.DfsWalProvider":{"logCloser":"org.apache.accumulo.server.log.NoOpLogCloser"}} ``` Closer implementations must support repeated calls and concurrent preparation of different WALs. @@ -78,19 +78,20 @@ them. Provider IDs and configuration must remain consistent across server proces The [11-to-12 upgrader][upgrader] assigns the built-in provider ID `dfs` to pre-SPI filesystem WALs without moving or rewriting the WAL files: -- Legacy tablet log qualifiers `-/` become `dfs/`: the upgrader adds the new - qualifier and deletes the old one. This covers the root tablet metadata stored in ZooKeeper, - metadata-tablet references, and user-tablet references. Already converted qualifiers are left - alone. +- Legacy tablet log qualifiers `-/` become `dfs/@/`: the upgrader + assigns the `dfs` provider ID and extracts the creator address from the legacy WAL path. It adds + the new qualifier and deletes the old one. This covers root tablet metadata stored in ZooKeeper, + metadata-tablet references, and user-tablet references. Already converted qualifiers are + validated and left alone. - Legacy ZooKeeper WAL marker data `,` becomes `,dfs,`. Markers already containing a provider ID are retained. The upgrader checks marker format and verifies that an updated marker was persisted. - Legacy recovery work `source|destination` becomes `sortId|dfs|source|destination`, using the recovery node's UUID as `sortId`. Converted work is validated and retained on retries; the `locks` node is skipped. Invalid provider IDs, blank paths, mismatched IDs, or failed writes stop the upgrade. -- The upgrader checks for remaining legacy `-/` log qualifiers in the metadata tables; it also - verifies the root tablet's WAL references. Malformed markers or leftover legacy qualifiers cause - the upgrade to fail rather than silently treating them as another provider's logs. +- The upgrader verifies that root, metadata-tablet, and user-tablet WAL qualifiers all use the new + format. Malformed markers or leftover legacy qualifiers cause the upgrade to fail rather than + silently treating them as another provider's logs. Configure `DfsWalProvider` in `instance.wal.providers` during the upgrade and retain it while old WAL references or markers exist, even if a different class is selected in `instance.wal.provider` @@ -111,7 +112,7 @@ recovery behavior. | Method | Use | | ----------------------------------- | ----------------------------------------------------- | -| `create(serverAddress, UUID)` | Exclusively create a WAL and return its `Writer`. | +| `create(UUID)` | Exclusively create a WAL and return its `Writer`. | | `prepareForRecovery(reference)` | Make an abandoned WAL consistently readable for sorting. | | `open(reference)` | Read a WAL from byte zero for sorting/recovery. | | `exists(reference)` | Check a WAL, including while it may still be written. | @@ -134,12 +135,12 @@ TabletServer process failure, `SYNC` must provide the synchronization guarantee WAL writes, and `LOG` does not invoke `Writer.sync()`. Closing a writer must be idempotent. The reference returned by `Writer.getReference()` is persisted and must stay usable by that -provider's `open()` and `delete()`. It must be a fully qualified Hadoop path in the standard WAL -layout, with a final `/` suffix (the address's `:` becomes `+`). The current -[recovery path conversion][rpath] also expects a `wal` directory immediately before the -server component; it maps `/wal//` to `/recovery/`. A custom -provider can use a custom Hadoop filesystem URI, but this path structure and access to the -corresponding recovery destination are current integration constraints. +provider's `open()` and `delete()`. It must be a fully qualified Hadoop path ending in the UUID +passed to `create()`. The built-in provider uses `/wal/`; legacy WALs may still use +`/wal//`. The [recovery path conversion][rpath] accepts both layouts and +maps them to `/recovery/`. A custom provider can use a custom Hadoop filesystem URI, +but the WAL directory and corresponding recovery destination must be accessible through +`VolumeManager`. ## Writing and tracking a WAL @@ -152,14 +153,15 @@ corresponding recovery destination are current integration constraints. compaction events use the existing WAL serialization. A background sync task batches durability requests and calls `Writer.sync(FLUSH)` or `Writer.sync(SYNC)`; after slow syncs it calls `getDiagnostics()` and `isHealthy()`. -3. `LogEntry` associates the provider ID with the path. Tablet metadata stores it as a log-column - qualifier of the form `/`. [`WalStateManager`][wal-state] records the WAL - state (`OPEN`, `CLOSED`, or `UNREFERENCED`), provider ID, and path in ZooKeeper WAL markers. This - lets recovery and GC identify which provider created a given log, independently of the provider - currently selected for new writes. +3. `LogEntry` associates the provider ID, creator address, and path. Tablet metadata stores them + as a log-column qualifier of the form `/@/`. + [`WalStateManager`][wal-state] records the WAL state (`OPEN`, `CLOSED`, or `UNREFERENCED`), + provider ID, and path in ZooKeeper WAL markers; the marker's parent node identifies the + TabletServer. This lets recovery and GC identify which provider created a given log, + independently of the provider currently selected for new writes. [`DfsWalProvider`][dfs-provider] is the built-in implementation. It chooses an Accumulo volume and -creates a filesystem-backed WAL under `wal//` using `VolumeManager`; its writer +creates a filesystem-backed WAL under `wal/` using `VolumeManager`; its writer uses `hflush()` for `FLUSH` and `hsync()` for `SYNC`. It opens and checks files through `VolumeManager`, and uses the filesystem trash or recursive deletion for cleanup. Where the underlying stream is a replicated HDFS `DFSOutputStream`, its writer records the observed replication diff --git a/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java index 7bdf5a8e591..469c1d4171f 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/wal/WalProvider.java @@ -41,8 +41,9 @@ * *

* To keep Accumulo's existing recovery pipeline, log references must be fully qualified Hadoop - * paths in the standard WAL layout, ending in {@code /}. Providers may use a - * custom Hadoop filesystem URI, but must preserve this path layout. + * paths ending in the WAL UUID. Providers may use a custom Hadoop filesystem URI, but the path must + * remain under the standard {@code wal} directory so Accumulo can locate the corresponding recovery + * directory. * * @since 4.0.0 */ @@ -93,17 +94,15 @@ interface InitParameters { long prepareForRecovery(String reference) throws IOException; /** - * Creates a new exclusive WAL for the given TabletServer and UUID. The returned reference must be - * unique, fully qualified, and end in the supplied server address (with {@code :} replaced by - * {@code +}) followed by the supplied UUID. The writer must preserve every byte written to its - * output stream in order. + * Creates a new exclusive WAL for the given UUID. The returned reference must be unique, fully + * qualified, and end with the supplied UUID as its final path component. The writer must preserve + * every byte written to its output stream in order. * - * @param serverAddress advertised TabletServer address in {@code host:port} form * @param id globally unique identifier for the WAL * @return a new writer whose reference can later be passed to {@link #open(String)} and * {@link #delete(String)} */ - Writer create(String serverAddress, UUID id) throws IOException; + Writer create(UUID id) throws IOException; /** * Opens a completed or abandoned WAL for recovery. The returned stream must start at byte zero diff --git a/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java b/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java index b66ca1c4845..b052a6e0ab0 100644 --- a/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java +++ b/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java @@ -31,49 +31,48 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.net.HostAndPort; public final class LogEntry { private static final String LOG_PATH = - "%s" + Path.SEPARATOR + Constants.WAL_DIR + Path.SEPARATOR + "%s" + Path.SEPARATOR + "%s"; + "%s" + Path.SEPARATOR + Constants.WAL_DIR + Path.SEPARATOR + "%s"; + private static final String ADDRESS_QUALIFIER_PREFIX = "@"; private final Path path; private final String providerId; - private final HostAndPort tserver; + private final @Nullable HostAndPort tserverAddress; private final UUID uniqueId; private final Text columnQualifier; - private LogEntry(Path path, String providerId, HostAndPort tserver, UUID uniqueId, - Text columnQualifier) { + private LogEntry(Path path, String providerId, @Nullable HostAndPort tserverAddress, + UUID uniqueId, Text columnQualifier) { this.path = path; this.providerId = providerId; - this.tserver = tserver; + this.tserverAddress = tserverAddress; this.uniqueId = uniqueId; this.columnQualifier = columnQualifier; } public static LogEntry withSwitchedVolume(LogEntry entry, Path newPath) { - return new LogEntry(validatePath(newPath), entry.getProviderId(), entry.getTServer(), - entry.getUniqueID(), entry.getColumnQualifier()); + return new LogEntry(validatePath(newPath), entry.getProviderId(), entry.getTServerAddress(), + entry.getUniqueID(), entry.columnQualifier); } public static LogEntry forServer(String providerId, String volume, HostAndPort address) { - UUID filename = UUID.randomUUID(); - String addressForFilename = address.toString().replace(':', '+'); - String filePath = LOG_PATH.formatted(volume, addressForFilename, filename); - return fromPath(providerId, filePath); + UUID id = UUID.randomUUID(); + String filePath = LOG_PATH.formatted(volume, id); + return fromPath(providerId, filePath, address); } /** - * Creates a new LogEntry object after validating the expected format of the path. The path must - * be an absolute filesystem URI and contain a tserver (host+port) followed by a UUID as the file - * name as the last two components.
- * For example, file:///some/dir/path/localhost+1234/927ba659-d109-4bce-b0a5-bcbbcb9942a2 is a - * valid path. + * Creates a new LogEntry object after validating the path. The path must be an absolute + * filesystem URI whose final component is a UUID. When possible, the TabletServer address is + * inferred from the penultimate component of legacy WAL paths; new references do not need to + * contain it. * * @param providerId provider that owns this WAL * @param path path to validate @@ -81,36 +80,28 @@ public static LogEntry forServer(String providerId, String volume, HostAndPort a * @throws IllegalArgumentException if the path is invalid */ public static LogEntry fromPath(String providerId, String path) { - return validatedLogEntry(path, providerId, null); + return validatedLogEntry(path, providerId, getLegacyTserverAddress(path), null); } - private static LogEntry validatedLogEntry(String path, String providerId, Text columnQualifier) { + /** + * Creates a LogEntry using an explicitly known TabletServer address. + * + * @param providerId provider that owns this WAL + * @param path fully qualified reference returned by the provider + * @param tserverAddress TabletServer address that created this WAL + */ + public static LogEntry fromPath(String providerId, String path, HostAndPort tserverAddress) { + return validatedLogEntry(path, providerId, validateTserverAddress(tserverAddress), null); + } + + private static LogEntry validatedLogEntry(String path, String providerId, + @Nullable HostAndPort tserverAddress, Text columnQualifier) { Preconditions.checkArgument( providerId != null && providerId.matches(WalProvider.PROVIDER_ID_REGEX), "Invalid WAL provider ID: %s", providerId); - String[] parts = path.split("/"); - - if (parts.length < 2) { - throw new IllegalArgumentException( - "Invalid path format. The path should end with tserver/UUID."); - } - - String tserverPart = parts[parts.length - 2]; - String uuidPart = parts[parts.length - 1]; - - String badTServerMsg = - "Invalid tserver in path. Expected: host+port. Found '" + tserverPart + "'"; - if (tserverPart.contains(":") || !tserverPart.contains("+")) { - throw new IllegalArgumentException(badTServerMsg); - } - HostAndPort tserver; - try { - tserver = HostAndPort.fromString(tserverPart.replace("+", ":")); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException(badTServerMsg); - } - - String badUuidMsg = "Expected valid UUID. Found '" + uuidPart + "'"; + Path logPath = validatePath(new Path(path)); + String uuidPart = logPath.getName(); + String badUuidMsg = "Expected valid UUID as the final path component. Found '" + uuidPart + "'"; UUID uuid; try { uuid = UUID.fromString(uuidPart); @@ -120,9 +111,30 @@ private static LogEntry validatedLogEntry(String path, String providerId, Text c if (!uuid.toString().equals(uuidPart)) { throw new IllegalArgumentException(badUuidMsg); } + return new LogEntry(logPath, providerId, tserverAddress, uuid, columnQualifier); + } - Path logPath = validatePath(new Path(path)); - return new LogEntry(logPath, providerId, tserver, uuid, columnQualifier); + private static HostAndPort validateTserverAddress(HostAndPort address) { + Preconditions.checkArgument(address != null && address.hasPort(), + "TabletServer address must include a port"); + return address; + } + + @Nullable + private static HostAndPort getLegacyTserverAddress(String path) { + Path parent = new Path(path).getParent(); + if (parent == null) { + return null; + } + String address = parent.getName(); + if (!address.contains("+")) { + return null; + } + try { + return validateTserverAddress(HostAndPort.fromString(address.replace('+', ':'))); + } catch (IllegalArgumentException e) { + return null; + } } private static Path validatePath(Path path) { @@ -147,15 +159,31 @@ public static LogEntry fromMetaWalEntry(Entry entry) { "The provided metadata entry's column family is %s instead of %s", fam, LogColumnFamily.NAME); Text qualifier = entry.getKey().getColumnQualifier(); - String[] parts = qualifier.toString().split("/", 2); - Preconditions.checkArgument(parts.length == 2, "Malformed write-ahead log %s", qualifier); - return validatedLogEntry(parts[1], parts[0], qualifier); + String qualifierString = qualifier.toString(); + int providerSeparator = qualifierString.indexOf('/'); + Preconditions.checkArgument(providerSeparator > 0, "Malformed write-ahead log %s", qualifier); + String providerId = qualifierString.substring(0, providerSeparator); + String rest = qualifierString.substring(providerSeparator + 1); + Preconditions.checkArgument(rest.startsWith(ADDRESS_QUALIFIER_PREFIX), + "WAL qualifier does not contain a TabletServer address: %s", qualifier); + int addressSeparator = rest.indexOf('/', ADDRESS_QUALIFIER_PREFIX.length()); + Preconditions.checkArgument(addressSeparator > ADDRESS_QUALIFIER_PREFIX.length(), + "Malformed WAL TabletServer address in qualifier %s", qualifier); + HostAndPort address; + try { + address = validateTserverAddress(HostAndPort + .fromString(rest.substring(ADDRESS_QUALIFIER_PREFIX.length(), addressSeparator))); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Malformed WAL TabletServer address in qualifier " + qualifier, e); + } + String path = rest.substring(addressSeparator + 1); + return validatedLogEntry(path, providerId, address, qualifier); } - @NonNull - @VisibleForTesting - public HostAndPort getTServer() { - return tserver; + @Nullable + public HostAndPort getTServerAddress() { + return tserverAddress; } @NonNull @@ -202,7 +230,9 @@ public Text getColumnQualifier() { } private Text newCQ() { - return new Text(providerId + "/" + getPath()); + HostAndPort address = Objects.requireNonNull(tserverAddress, + "TabletServer address is required to write a WAL metadata qualifier"); + return new Text(providerId + "/" + ADDRESS_QUALIFIER_PREFIX + address + "/" + getPath()); } /** diff --git a/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java b/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java index 6e204c1ec56..2148f116004 100644 --- a/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java +++ b/core/src/test/java/org/apache/accumulo/core/tabletserver/log/LogEntryTest.java @@ -52,12 +52,13 @@ public void testColumnFamily() { @Test public void testFromPath() { var logEntry = LogEntry.fromPath(PROVIDER_ID, validPath); - verifyLogEntry(logEntry, new Text(PROVIDER_ID + "/" + validPath)); + verifyLogEntry(logEntry, + new Text(PROVIDER_ID + "/@" + validHost.replace('+', ':') + "/" + validPath)); } @Test public void testFromMetadata() { - var columnQualifier = "prefix/" + validPath; + var columnQualifier = PROVIDER_ID + "/@" + validHost.replace('+', ':') + "/" + validPath; var logEntry = LogEntry.fromMetaWalEntry( new SimpleImmutableEntry<>(new Key("1<", LogColumnFamily.STR_NAME, columnQualifier), null)); verifyLogEntry(logEntry, new Text(columnQualifier)); @@ -101,9 +102,8 @@ public void testForServer() { assertEquals(logEntry, roundTripped); assertEquals(logEntry.getPath(), roundTripped.getPath()); assertEquals(logEntry.getUniqueID(), roundTripped.getUniqueID()); - assertEquals(logEntry.getTServer(), roundTripped.getTServer()); - assertEquals(HostAndPort.fromString(validHost.replace('+', ':')), logEntry.getTServer()); - assertEquals("viewfs:/a/accumulo/wal/" + validHost + "/" + roundTripped.getUniqueID(), + assertEquals(HostAndPort.fromString(validHost.replace('+', ':')), logEntry.getTServerAddress()); + assertEquals("viewfs:/a/accumulo/wal/" + roundTripped.getUniqueID(), logEntry.getPath().toString()); } @@ -117,7 +117,7 @@ public void testForServerRequiresQualifiedVolume() { private void verifyLogEntry(LogEntry logEntry, Text expectedColumnQualifier) { assertEquals(validPath, logEntry.toString()); assertEquals(validPath, logEntry.getPath().toString()); - assertEquals(HostAndPort.fromString(validHost.replace('+', ':')), logEntry.getTServer()); + assertEquals(HostAndPort.fromString(validHost.replace('+', ':')), logEntry.getTServerAddress()); assertEquals(expectedColumnQualifier, logEntry.getColumnQualifier()); assertEquals(validUUID, logEntry.getUniqueID()); } @@ -130,7 +130,7 @@ public void testEquals() { assertNotSame(one, two); assertEquals(one.toString(), two.toString()); assertEquals(one.getPath(), two.getPath()); - assertEquals(one.getTServer(), two.getTServer()); + assertEquals(one.getTServerAddress(), two.getTServerAddress()); assertEquals(one.getColumnQualifier(), two.getColumnQualifier()); assertEquals(one.getUniqueID(), two.getUniqueID()); assertEquals(one, two); @@ -159,7 +159,7 @@ public void testRelativePathsAreInvalid() { @Test public void testRelativeMetadataPathIsInvalid() { - var qualifier = "prefix/" + validHost + "/" + validUUID; + var qualifier = "prefix/@" + validHost.replace('+', ':') + "/relative/" + validUUID; var entry = new SimpleImmutableEntry( new Key("1<", LogColumnFamily.STR_NAME, qualifier), null); assertThrows(IllegalArgumentException.class, () -> LogEntry.fromMetaWalEntry(entry)); @@ -167,30 +167,29 @@ public void testRelativeMetadataPathIsInvalid() { @Test public void testBadPathLength() { - Stream.of("foo", "", validHost).forEach(badPath -> { - var e = assertThrows(IllegalArgumentException.class, - () -> LogEntry.fromPath(PROVIDER_ID, badPath)); - assertTrue(e.getMessage().contains("The path should end with tserver/UUID.")); - }); + Stream.of("file:///accumulo/wal/foo", "file:///accumulo/wal/", "file:///accumulo/wal") + .forEach(badPath -> { + var e = assertThrows(IllegalArgumentException.class, + () -> LogEntry.fromPath(PROVIDER_ID, badPath)); + assertTrue(e.getMessage().contains("Expected valid UUID")); + }); } @Test - public void testInvalidHostPort() { - Stream.of("default:9997", "default+badPort").forEach(badHostAndPort -> { - var badPath = badHostAndPort + "/" + validUUID; - var e = assertThrows(IllegalArgumentException.class, - () -> LogEntry.fromPath(PROVIDER_ID, badPath)); - assertTrue(e.getMessage().contains("Expected: host+port. Found '" + badHostAndPort + "'")); - }); + public void testPathDoesNotRequireTserverAddress() { + var logEntry = + LogEntry.fromPath(PROVIDER_ID, "file:///accumulo/wal/provider-shard/" + validUUID); + assertEquals(validUUID, logEntry.getUniqueID()); + assertTrue(logEntry.getTServerAddress() == null); } @Test public void testInvalidUUID() { var badUUID = "badUUID"; - var pathWithBadUUID = validHost + "/" + badUUID; + var pathWithBadUUID = "file:///accumulo/wal/" + badUUID; var e = assertThrows(IllegalArgumentException.class, () -> LogEntry.fromPath(PROVIDER_ID, pathWithBadUUID)); - assertTrue(e.getMessage().contains("Expected valid UUID. Found '" + badUUID + "'")); + assertTrue(e.getMessage().contains("Expected valid UUID as the final path component")); } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java index c32e952f4df..caded9bbee4 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/DfsWalProvider.java @@ -81,13 +81,11 @@ public long prepareForRecovery(String reference) throws IOException { } @Override - public Writer create(String serverAddress, UUID id) throws IOException { - String addressForFilename = serverAddress.replace(':', '+'); + public Writer create(UUID id) throws IOException { var chooserEnv = new VolumeChooserEnvironmentImpl(VolumeChooserEnvironment.Scope.LOGGER, context); - String logPath = - context.getVolumeManager().choose(chooserEnv, context.getBaseUris()) + Path.SEPARATOR - + Constants.WAL_DIR + Path.SEPARATOR + addressForFilename + Path.SEPARATOR + id; + String logPath = context.getVolumeManager().choose(chooserEnv, context.getBaseUris()) + + Path.SEPARATOR + Constants.WAL_DIR + Path.SEPARATOR + id; var path = new Path(logPath); VolumeManager fs = context.getVolumeManager(); short replication = (short) context.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION); diff --git a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java index 118ef0f1197..90035b51f3e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/log/WalStateManager.java @@ -131,12 +131,13 @@ public void initWalMarker(TServerInstance tsi) throws WalMarkerException { } public void addNewWalMarker(TServerInstance tsi, LogEntry entry) throws WalMarkerException { - updateState(tsi, entry.getProviderId(), entry.getPath(), WalState.OPEN); + updateState(tsi, entry, WalState.OPEN); } - private void updateState(TServerInstance tsi, String providerId, Path path, WalState state) + private void updateState(TServerInstance tsi, LogEntry entry, WalState state) throws WalMarkerException { - byte[] data = (state + "," + providerId + "," + path).getBytes(UTF_8); + Path path = entry.getPath(); + byte[] data = (state + "," + entry.getProviderId() + "," + path).getBytes(UTF_8); try { NodeExistsPolicy policy = NodeExistsPolicy.OVERWRITE; if (state == WalState.OPEN) { @@ -153,11 +154,19 @@ private void updateState(TServerInstance tsi, String providerId, Path path, WalS } public void walUnreferenced(TServerInstance tsi, LogEntry entry) throws WalMarkerException { - updateState(tsi, entry.getProviderId(), entry.getPath(), WalState.UNREFERENCED); + updateState(tsi, entry, WalState.UNREFERENCED); } public void closeWal(TServerInstance tsi, LogEntry entry) throws WalMarkerException { - closeWal(tsi, entry.getPath(), entry.getProviderId()); + try { + state(tsi, UUID.fromString(entry.getPath().getName())); + } catch (WalMarkerException e) { + if (e.getCause() instanceof KeeperException.NoNodeException) { + return; + } + throw e; + } + updateState(tsi, entry, WalState.CLOSED); } public static WalStatePath parse(byte[] data) { @@ -189,7 +198,8 @@ public List getWalsInUse(TServerInstance tsi) throws WalMarkerExceptio if (zdata != null) { WalStatePath parts = parse(zdata); if (parts.state() != WalState.UNREFERENCED) { - result.add(LogEntry.fromPath(parts.providerId(), parts.path().toString())); + result.add(LogEntry.fromPath(parts.providerId(), parts.path().toString(), + tsi.getHostAndPort())); } } } @@ -289,25 +299,4 @@ public void forget(TServerInstance instance) throws WalMarkerException { } } - // tablet server can mark the log as closed (but still needed) - // manager can mark a log as unreferenced after it has made log recovery markers on the tablets - // that need to be recovered - public void closeWal(TServerInstance instance, Path path) throws WalMarkerException { - closeWal(instance, path, null); - } - - private void closeWal(TServerInstance instance, Path path, String providerId) - throws WalMarkerException { - WalStatePath existing; - try { - existing = state(instance, UUID.fromString(path.getName())); - } catch (WalMarkerException e) { - if (e.getCause() instanceof KeeperException.NoNodeException) { - return; - } - throw e; - } - updateState(instance, providerId == null ? existing.providerId() : providerId, path, - WalState.CLOSED); - } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/recovery/RecoveryPath.java b/server/base/src/main/java/org/apache/accumulo/server/manager/recovery/RecoveryPath.java index 893867bad26..711ced14f83 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/recovery/RecoveryPath.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/recovery/RecoveryPath.java @@ -18,41 +18,44 @@ */ package org.apache.accumulo.server.manager.recovery; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.server.fs.VolumeManager.FileType; import org.apache.hadoop.fs.Path; public class RecoveryPath { - // given a wal path, transform it to a recovery path - public static Path getRecoveryPath(Path walPath) { - if (walPath.depth() >= 3 && walPath.toUri().getScheme() != null) { - // its a fully qualified path - String uuid = walPath.getName(); - // drop uuid - walPath = walPath.getParent(); + // given a WAL entry, transform it to a recovery path + public static Path getRecoveryPath(LogEntry logEntry) { + Path walPath = logEntry.getPath(); + if (walPath.toUri().getScheme() == null || !walPath.isAbsolute() + || !walPath.getName().equals(logEntry.getUniqueID().toString())) { + throw new IllegalArgumentException("Bad WAL path " + walPath); + } - // expect and drop the server component - if (walPath.getName().equals(FileType.WAL.getDirectory())) { - throw new IllegalArgumentException("Bath path " + walPath + " (missing server component)"); - } - walPath = walPath.getParent(); + Path parent = walPath.getParent(); + if (parent == null) { + throw new IllegalArgumentException("Bad WAL path " + walPath); + } - // expect and drop the wal component - if (!walPath.getName().equals(FileType.WAL.getDirectory())) { - throw new IllegalArgumentException( - "Bad path " + walPath + " (missing wal directory component)"); + Path walDir; + if (parent.getName().equals(FileType.WAL.getDirectory())) { + // New paths are /wal/. + walDir = parent; + } else { + // Legacy paths are /wal//. The tserver component is not needed + // to determine the recovery path. + walDir = parent.getParent(); + if (walDir == null || !walDir.getName().equals(FileType.WAL.getDirectory())) { + throw new IllegalArgumentException("Bad WAL path " + walPath); } - walPath = walPath.getParent(); - - // create new path in recovery directory that is a sibling to the wal directory (same volume), - // without the server component - walPath = new Path(walPath, FileType.RECOVERY.getDirectory()); - walPath = new Path(walPath, uuid); - - return walPath; } - throw new IllegalArgumentException("Bad path " + walPath); + Path volume = walDir.getParent(); + if (volume == null) { + throw new IllegalArgumentException("Bad WAL path " + walPath); + } + return new Path(new Path(volume, FileType.RECOVERY.getDirectory()), + logEntry.getUniqueID().toString()); } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/log/WalStateManagerTest.java b/server/base/src/test/java/org/apache/accumulo/server/log/WalStateManagerTest.java index 90919873aae..b75baefc464 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/log/WalStateManagerTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/log/WalStateManagerTest.java @@ -71,8 +71,7 @@ void getWalsInUsePreservesProviderId() throws Exception { ZooReaderWriter zoo = createMock(ZooReaderWriter.class); TServerInstance server = new TServerInstance("localhost:1234[SESSION]"); String serverPath = WalStateManager.ZWALS + "/" + server; - String reference = - "customfs://nn/accumulo/wal/localhost+1234/11111111-1111-1111-1111-111111111111"; + String reference = "customfs://nn/accumulo/wal/11111111-1111-1111-1111-111111111111"; expect(context.getZooSession()).andReturn(session); expect(session.asReaderWriter()).andReturn(zoo); @@ -85,7 +84,9 @@ void getWalsInUsePreservesProviderId() throws Exception { replay(context, session, zoo); WalStateManager manager = new WalStateManager(context); - assertEquals(List.of(LogEntry.fromPath("custom", reference)), manager.getWalsInUse(server)); + var wal = manager.getWalsInUse(server).get(0); + assertEquals(LogEntry.fromPath("custom", reference, server.getHostAndPort()), wal); + assertEquals(server.getHostAndPort(), wal.getTServerAddress()); verify(context, session, zoo); } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/recovery/RecoveryPathTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/recovery/RecoveryPathTest.java new file mode 100644 index 00000000000..54489a62d8e --- /dev/null +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/recovery/RecoveryPathTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.manager.recovery; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.apache.accumulo.core.tabletserver.log.LogEntry; +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.Test; + +class RecoveryPathTest { + + private static final String ID = "11111111-1111-1111-1111-111111111111"; + + @Test + void mapsNewWalLayout() { + var log = LogEntry.fromPath("dfs", "hdfs://nn/accumulo/wal/" + ID); + assertEquals(new Path("hdfs://nn/accumulo/recovery/" + ID), RecoveryPath.getRecoveryPath(log)); + } + + @Test + void mapsLegacyWalLayout() { + var log = LogEntry.fromPath("dfs", "hdfs://nn/accumulo/wal/host+1234/" + ID); + assertEquals(new Path("hdfs://nn/accumulo/recovery/" + ID), RecoveryPath.getRecoveryPath(log)); + } + + @Test + void requiresUuidAsBasenameAndWalDirectory() { + var wrongWalDirectory = LogEntry.fromPath("dfs", "hdfs://nn/accumulo/other/" + ID); + assertThrows(IllegalArgumentException.class, + () -> RecoveryPath.getRecoveryPath(wrongWalDirectory)); + + var original = LogEntry.fromPath("dfs", "hdfs://nn/accumulo/wal/" + ID); + var wrongUuid = LogEntry.withSwitchedVolume(original, + new Path("hdfs://nn/accumulo/wal/22222222-2222-2222-2222-222222222222")); + assertThrows(IllegalArgumentException.class, () -> RecoveryPath.getRecoveryPath(wrongUuid)); + } +} diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java index a8cd59d9efc..aa1d0eee138 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/recovery/RecoveryManager.java @@ -86,28 +86,23 @@ public RecoveryManager(Manager manager, long timeToCacheExistsInMillis) { } private class LogSortTask implements Runnable { - private final String source; - private final String destination; - private final UUID sortId; - private final String providerId; - private final WalProvider provider; - - public LogSortTask(WalProvider provider, String providerId, String source, String destination, - UUID sortId) { - this.provider = provider; - this.providerId = providerId; - this.source = source; - this.destination = destination; - this.sortId = sortId; + private final LogEntry logEntry; + + public LogSortTask(LogEntry logEntry) { + this.logEntry = logEntry; } @Override public void run() { boolean rescheduled = false; try { + WalProvider provider = + manager.getContext().getWalProviderRegistry().get(logEntry.getProviderId()); + String source = logEntry.getPath().toString(); long time = provider.prepareForRecovery(source); if (time < 0) { - throw new IllegalArgumentException("Negative recovery retry delay from " + providerId); + throw new IllegalArgumentException( + "Negative recovery retry delay from " + logEntry.getProviderId()); } if (time > 0) { @@ -115,16 +110,16 @@ public void run() { ThreadPools.watchNonCriticalScheduledTask(future); rescheduled = true; } else { - initiateSort(sortId, providerId, source, destination); + initiateSort(logEntry); } } catch (FileNotFoundException e) { - log.debug("Unable to initiate log sort for " + source + ": " + e); + log.debug("Unable to initiate log sort for {}: {}", logEntry.getPath(), e); } catch (Exception e) { - log.warn("Failed to initiate log sort " + source, e); + log.warn("Failed to initiate log sort {}", logEntry.getPath(), e); } finally { if (!rescheduled) { synchronized (RecoveryManager.this) { - closeTasksQueued.remove(sortId.toString()); + closeTasksQueued.remove(logEntry.getUniqueID().toString()); } } } @@ -132,14 +127,16 @@ public void run() { } - private void initiateSort(UUID sortId, String providerId, String source, final String destination) - throws KeeperException, InterruptedException { - String work = new LogSortInformation(sortId, providerId, source, destination).serialize(); + private void initiateSort(LogEntry logEntry) throws KeeperException, InterruptedException { + String sortId = logEntry.getUniqueID().toString(); + String destination = RecoveryPath.getRecoveryPath(logEntry).toString(); + String work = new LogSortInformation(logEntry.getUniqueID(), logEntry.getProviderId(), + logEntry.getPath().toString(), destination).serialize(); new DistributedWorkQueue(Constants.ZRECOVERY, manager.getConfiguration(), manager) - .addWork(sortId.toString(), work.getBytes(UTF_8)); + .addWork(sortId, work.getBytes(UTF_8)); synchronized (this) { - sortsQueued.add(sortId.toString()); + sortsQueued.add(sortId); } log.info("Created zookeeper entry {} with data {}", Constants.ZRECOVERY + "/" + sortId, work); @@ -200,7 +197,7 @@ private boolean recoverLog(LogEntry walog) throws IOException { UUID sortId = walog.getUniqueID(); String sortIdString = sortId.toString(); String filename = walog.getPath().toString(); - String dest = RecoveryPath.getRecoveryPath(walog.getPath()).toString(); + String dest = RecoveryPath.getRecoveryPath(walog).toString(); boolean sortQueued; synchronized (this) { @@ -229,8 +226,6 @@ private boolean recoverLog(LogEntry walog) throws IOException { synchronized (this) { if (!closeTasksQueued.contains(sortIdString) && !sortsQueued.contains(sortIdString)) { AccumuloConfiguration aconf = manager.getConfiguration(); - WalProvider provider = - manager.getContext().getWalProviderRegistry().get(walog.getProviderId()); Long delay = recoveryDelay.get(sortIdString); if (delay == null) { delay = aconf.getTimeInMillis(Property.MANAGER_RECOVERY_DELAY); @@ -240,9 +235,8 @@ private boolean recoverLog(LogEntry walog) throws IOException { log.info("Starting recovery of {} (in : {}s)", filename, (delay / 1000)); - ScheduledFuture future = executor.schedule( - new LogSortTask(provider, walog.getProviderId(), filename, dest, sortId), delay, - TimeUnit.MILLISECONDS); + ScheduledFuture future = + executor.schedule(new LogSortTask(walog), delay, TimeUnit.MILLISECONDS); ThreadPools.watchNonCriticalScheduledTask(future); closeTasksQueued.add(sortIdString); recoveryDelay.put(sortIdString, delay); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java index 5aa5bd97f53..ded20869ced 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java @@ -122,6 +122,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.net.HostAndPort; import com.google.gson.JsonParser; //TODO when removing this class, also remove MetadataSchema.Upgrader11to12 @@ -924,20 +925,28 @@ private static void upgradeWalEntry(Entry entry, Value value, Mutatio int separator = qualifierString.indexOf('/'); Preconditions.checkArgument(separator > 0, "Malformed write-ahead log %s", qualifierString); String providerId = qualifierString.substring(0, separator); + String encodedReference = qualifierString.substring(separator + 1); + if (encodedReference.startsWith("@")) { + // This WAL qualifier may already have been converted if an earlier upgrade attempt failed + // partway through. Validate it and leave it unchanged so the operation is safe to retry. + LogEntry.fromMetaWalEntry(entry); + return; + } if (providerId.equals("-")) { providerId = DfsWalProvider.PROVIDER_ID; } - String reference = qualifierString.substring(separator + 1); + String reference = encodedReference; + HostAndPort tserverAddress = getLegacyTserverAddress(new Path(reference)); LogEntry logEntry; try { - logEntry = LogEntry.fromPath(providerId, reference); + logEntry = LogEntry.fromPath(providerId, reference, tserverAddress); } catch (IllegalArgumentException e) { if (!providerId.equals(DfsWalProvider.PROVIDER_ID) || !e.getMessage() .startsWith("Invalid path format. The path must be an absolute URI: ")) { throw e; } Path qualifiedPath = qualifyWalPath(new Path(reference), volumes); - logEntry = LogEntry.fromPath(providerId, qualifiedPath.toString()); + logEntry = LogEntry.fromPath(providerId, qualifiedPath.toString(), tserverAddress); } Text newQualifier = logEntry.getColumnQualifier(); if (!newQualifier.equals(qualifier)) { @@ -946,6 +955,19 @@ private static void upgradeWalEntry(Entry entry, Value value, Mutatio } } + private static HostAndPort getLegacyTserverAddress(Path walPath) { + Path parent = walPath.getParent(); + Preconditions.checkArgument(parent != null, + "Legacy WAL path must end with TabletServer and UUID: %s", walPath); + String encodedAddress = parent.getName(); + Preconditions.checkArgument(encodedAddress.contains("+"), + "Legacy WAL path is missing its TabletServer address: %s", walPath); + HostAndPort address = HostAndPort.fromString(encodedAddress.replace('+', ':')); + Preconditions.checkArgument(address.hasPort(), + "Legacy WAL path has an invalid TabletServer address: %s", walPath); + return address; + } + private static Path qualifyWalPath(Path walPath, Collection volumes) throws IOException { String path = walPath.toUri().getPath(); String[] parts = path.split("/"); @@ -1048,10 +1070,13 @@ void processReferences(MutationWriter batchWriter, Iterable } private static void verifyNoLegacyWalReference(Key key, String source) { - if (key.getColumnFamily().equals(LogColumnFamily.NAME) - && key.getColumnQualifier().toString().startsWith("-/")) { - throw new IllegalStateException( - "Legacy WAL reference remains after upgrade in " + source + ": " + key); + if (key.getColumnFamily().equals(LogColumnFamily.NAME)) { + try { + LogEntry.fromMetaWalEntry(Map.entry(key, new Value())); + } catch (IllegalArgumentException e) { + throw new IllegalStateException( + "Legacy or malformed WAL reference remains after upgrade in " + source + ": " + key, e); + } } } diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java index 87fec9cdd1f..ccfe0b5923b 100644 --- a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java +++ b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java @@ -223,12 +223,12 @@ void upgradeWalReferenceTest() { assertEquals(2, updates.size()); var added = updates.stream().filter(update -> !update.isDeleted()).findFirst().orElseThrow(); var deleted = updates.stream().filter(ColumnUpdate::isDeleted).findFirst().orElseThrow(); - assertEquals(DfsWalProvider.PROVIDER_ID + "/" + path, - new Text(added.getColumnQualifier()).toString()); + String newQualifier = DfsWalProvider.PROVIDER_ID + "/@host:1234/" + path; + assertEquals(newQualifier, new Text(added.getColumnQualifier()).toString()); assertEquals("-/" + path, new Text(deleted.getColumnQualifier()).toString()); - Key upgradedKey = Key.builder().row("123").family(LogColumnFamily.NAME) - .qualifier(DfsWalProvider.PROVIDER_ID + "/" + path).build(); + Key upgradedKey = + Key.builder().row("123").family(LogColumnFamily.NAME).qualifier(newQualifier).build(); ArrayList retryMutations = new ArrayList<>(); new Upgrader11to12().processReferences(retryMutations::add, List.of(Map.entry(upgradedKey, value)), "accumulo.metadata", List.of()); @@ -305,7 +305,7 @@ public void processReferencesQualifiesWalPathsUsingVolume(@TempDir java.nio.file .qualifier(oldQualifier).build(); scanData.put(walKey, new Value("wal-value")); oldQualifiers.add(oldQualifier); - expectedQualifiers.add(DfsWalProvider.PROVIDER_ID + "/" + existingWal); + expectedQualifiers.add(DfsWalProvider.PROVIDER_ID + "/@localhost:9997/" + existingWal); } ArrayList mutations = new ArrayList<>(); @@ -947,7 +947,7 @@ void testRootWalUpgradePersistsAndRetries() throws Exception { upgrader.updateRootTabletFileReferences(context); var entries = new RootTabletMetadata(new String(persisted.getValue(), UTF_8)).toKeyValues(); assertEquals(1, entries.size()); - assertEquals("dfs/" + reference, entries.firstKey().getColumnQualifier().toString()); + assertEquals("dfs/@host:1234/" + reference, entries.firstKey().getColumnQualifier().toString()); verify(context, volumes, session, zoo); } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java index 0f5df41fea0..fe9106eac3e 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java @@ -74,6 +74,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; +import com.google.common.net.HostAndPort; /** * Wrap a connection to a logger. @@ -287,17 +288,22 @@ public int hashCode() { /** * Create a new DfsLogger with the provided characteristics. */ - public static DfsLogger createNew(ServerContext context, WalProvider provider, - AtomicLong syncCounter, AtomicLong flushCounter, String address) throws IOException { + public static DfsLogger createNew(ServerContext context, AtomicLong syncCounter, + AtomicLong flushCounter, HostAndPort address, WalProvider provider) throws IOException { - WalProvider.Writer writer = provider.create(address, UUID.randomUUID()); + UUID id = UUID.randomUUID(); + WalProvider.Writer writer = provider.create(id); DfsLogger dfsLogger = null; try { - LogEntry log = LogEntry.fromPath(provider.getId(), writer.getReference()); + LogEntry log = LogEntry.fromPath(provider.getId(), writer.getReference(), address); + if (!id.equals(log.getUniqueID())) { + throw new IllegalArgumentException("WAL reference " + writer.getReference() + + " does not end with the requested UUID " + id); + } dfsLogger = new DfsLogger(log); long slowFlushMillis = context.getConfiguration().getTimeInMillis(Property.TSERV_SLOW_FLUSH_MILLIS); - dfsLogger.open(context, writer, address, syncCounter, flushCounter, slowFlushMillis); + dfsLogger.open(context, writer, syncCounter, flushCounter, slowFlushMillis); return dfsLogger; } catch (IOException | RuntimeException e) { try { @@ -381,9 +387,9 @@ public static DataInputStream getDecryptingStream(InputStream input, CryptoServi * exception is thrown from this method, it is the callers responsibility to ensure that * {@link #close()} is called to prevent leaking the file handle and/or syncing thread. */ - private synchronized void open(ServerContext context, WalProvider.Writer writer, String address, + private synchronized void open(ServerContext context, WalProvider.Writer writer, AtomicLong syncCounter, AtomicLong flushCounter, long slowFlushMillis) throws IOException { - log.debug("Address is {}", address); + log.debug("Address is {}", logEntry.getTServerAddress()); log.debug("DfsLogger.open() begin"); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java index 7a328ff6d9b..d398d0bdb95 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java @@ -135,7 +135,7 @@ public void process(String child, byte[] data) { String formerThreadName = Thread.currentThread().getName(); try { - sort(fs, sortId, providerId, new Path(src), dest); + sort(fs, sortId, new Path(src), dest, providerId); } catch (Exception t) { try { // parent dir may not exist @@ -159,8 +159,8 @@ public void process(String child, byte[] data) { } } - public void sort(VolumeManager fs, String name, String providerId, Path srcPath, - String destPath) throws IOException { + public void sort(VolumeManager fs, String name, Path srcPath, String destPath, + String providerId) throws IOException { int part = 0; // check for finished first since another thread may have already done the sort diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/ResolvedSortedLog.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/ResolvedSortedLog.java index 1b87af64666..b0babe1793c 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/ResolvedSortedLog.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/ResolvedSortedLog.java @@ -87,7 +87,7 @@ public static ResolvedSortedLog resolve(LogEntry logEntry, VolumeManager fs) thr // convert the path of an unsorted log to the expected path for the corresponding sorted log // dir - Path sortedLogPath = RecoveryPath.getRecoveryPath(logEntry.getPath()); + Path sortedLogPath = RecoveryPath.getRecoveryPath(logEntry); boolean foundFinish = false; // Path::getName compares the last component of each Path value. In this case, the last diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java index fa9c97603b8..40f5173aa4a 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java @@ -277,9 +277,9 @@ private synchronized void startLogMaker() { final DfsLogger alog; try { - alog = DfsLogger.createNew(tserver.getContext(), - tserver.getContext().getWalProviderRegistry().getCurrent(), syncCounter, flushCounter, - tserver.getAdvertiseAddress().toString()); + alog = DfsLogger.createNew(tserver.getContext(), syncCounter, flushCounter, + tserver.getAdvertiseAddress(), + tserver.getContext().getWalProviderRegistry().getCurrent()); } catch (IOException | RuntimeException e) { log.error("Failed to open WAL", e); try { diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java index 1f5b561cd7a..019b41f1838 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java @@ -116,8 +116,8 @@ public void testUpgradeOf15WALog() throws IOException { LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor(); assertThrows(IllegalArgumentException.class, - () -> logProcessor.sort(context.getVolumeManager(), "/" + WALOG_FROM_15, "dfs", - new Path(testPath.toUri() + WALOG_FROM_15), testPath.toUri() + "manyMaps")); + () -> logProcessor.sort(context.getVolumeManager(), "/" + WALOG_FROM_15, + new Path(testPath.toUri() + WALOG_FROM_15), testPath.toUri() + "manyMaps", "dfs")); } @Test @@ -132,8 +132,8 @@ public void testBasic16WALogRead() throws IOException { LogSorter logSorter = new LogSorter(server); LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor(); - logProcessor.sort(context.getVolumeManager(), "/" + walogToTest, "dfs", - new Path(testPath.toUri() + walogToTest), destPath); + logProcessor.sort(context.getVolumeManager(), "/" + walogToTest, + new Path(testPath.toUri() + walogToTest), destPath, "dfs"); assertTrue(context.getVolumeManager().exists(getFinishedMarkerPath(destPath))); } @@ -149,8 +149,8 @@ public void testBasic20WALogRead() throws IOException { LogSorter logSorter = new LogSorter(server); LogSorter.LogProcessor logProcessor = logSorter.new LogProcessor(); - logProcessor.sort(context.getVolumeManager(), "/" + walogToTest, "dfs", - new Path(testPath.toUri() + walogToTest), destPath); + logProcessor.sort(context.getVolumeManager(), "/" + walogToTest, + new Path(testPath.toUri() + walogToTest), destPath, "dfs"); assertTrue(context.getVolumeManager().exists(getFinishedMarkerPath(destPath))); } diff --git a/test/src/main/java/org/apache/accumulo/test/RecoveryIT.java b/test/src/main/java/org/apache/accumulo/test/RecoveryIT.java index fd4642e3aec..73d441277d7 100644 --- a/test/src/main/java/org/apache/accumulo/test/RecoveryIT.java +++ b/test/src/main/java/org/apache/accumulo/test/RecoveryIT.java @@ -224,7 +224,7 @@ private boolean logSortingCompleted(AccumuloClient c, TableId tableId) throws Ex for (TabletMetadata tm : tabletsMetadata) { for (LogEntry walog : tm.getLogs()) { String sortId = walog.getUniqueID().toString(); - String dest = RecoveryPath.getRecoveryPath(walog.getPath()).toString(); + String dest = RecoveryPath.getRecoveryPath(walog).toString(); if (ctx.getZooCache().get(Constants.ZRECOVERY + "/" + sortId) != null || !ctx.getVolumeManager().exists(SortedLogState.getFinishedMarkerPath(dest))) { diff --git a/test/src/main/java/org/apache/accumulo/test/upgrade/UpgradeUtilIT.java b/test/src/main/java/org/apache/accumulo/test/upgrade/UpgradeUtilIT.java index c4c911746ba..ee285c68612 100644 --- a/test/src/main/java/org/apache/accumulo/test/upgrade/UpgradeUtilIT.java +++ b/test/src/main/java/org/apache/accumulo/test/upgrade/UpgradeUtilIT.java @@ -29,6 +29,7 @@ import java.util.UUID; import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.fate.zookeeper.ZooReader; @@ -37,6 +38,7 @@ import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeExistsPolicy; import org.apache.accumulo.core.volume.Volume; import org.apache.accumulo.minicluster.ServerType; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.server.AccumuloDataVersion; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.ServerDirs; @@ -44,6 +46,7 @@ import org.apache.accumulo.server.util.upgrade.UpgradeProgress; import org.apache.accumulo.test.harness.AccumuloClusterHarness; import org.apache.accumulo.test.util.Wait; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.zookeeper.CreateMode; @@ -54,6 +57,11 @@ public class UpgradeUtilIT extends AccumuloClusterHarness { private static final String ZTABLE_NAME = "/name"; + @Override + public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { + cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "10s"); + } + @BeforeEach public void beforeTest() throws Exception { ServerContext ctx = getCluster().getServerContext(); diff --git a/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java index e04e642697b..26b4675b9cf 100644 --- a/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java +++ b/test/src/main/java/org/apache/accumulo/test/wal/InMemoryWalProvider.java @@ -91,15 +91,13 @@ public long prepareForRecovery(String reference) throws IOException { } @Override - public Writer create(String serverAddress, UUID id) throws IOException { + public Writer create(UUID id) throws IOException { requireInitialized(); - String hostForPath = Objects.requireNonNull(serverAddress).replace(':', '+'); String reference; try { - reference = - new URI(SCHEME, null, referenceRoot() + hostForPath + "/" + id, null, null).toString(); + reference = new URI(SCHEME, null, referenceRoot() + id, null, null).toString(); } catch (URISyntaxException e) { - throw new IOException("Invalid TabletServer address for WAL reference: " + serverAddress, e); + throw new IOException("Invalid WAL UUID for reference: " + id, e); } validateReference(reference); try (var socket = InMemoryWalStore.connect(host, port); @@ -179,10 +177,10 @@ private void validateReference(String reference) throws IOException { throw new IOException("Invalid in-memory WAL reference: " + reference); } String[] parts = path.substring(referenceRoot().length()).split("/", -1); - if (parts.length != 2 || parts[0].isBlank() || parts[1].isBlank()) { + if (parts.length != 1 || parts[0].isBlank()) { throw new IOException("Invalid in-memory WAL reference: " + reference); } - UUID.fromString(parts[1]); + UUID.fromString(parts[0]); } catch (URISyntaxException | IllegalArgumentException e) { throw new IOException("Invalid in-memory WAL reference: " + reference, e); } diff --git a/test/src/test/java/org/apache/accumulo/test/wal/InMemoryWalProviderTest.java b/test/src/test/java/org/apache/accumulo/test/wal/InMemoryWalProviderTest.java index b7852b44ba0..4e799641305 100644 --- a/test/src/test/java/org/apache/accumulo/test/wal/InMemoryWalProviderTest.java +++ b/test/src/test/java/org/apache/accumulo/test/wal/InMemoryWalProviderTest.java @@ -43,12 +43,13 @@ void sharesSyncedAndClosedWalBytesAcrossProviders() throws Exception { WalProvider writerProvider = newProvider(store.getProviderOptions()); WalProvider readerProvider = newProvider(store.getProviderOptions()); UUID id = UUID.randomUUID(); - WalProvider.Writer writer = writerProvider.create("tserver:1234", id); + WalProvider.Writer writer = writerProvider.create(id); String reference = writer.getReference(); - assertEquals("file:/tmp/accumulo-inmemory-wal-" - + store.getProviderOptions().get(InMemoryWalProvider.PORT_OPTION) + "/wal/tserver+1234/" - + id, reference); + assertEquals( + "file:/tmp/accumulo-inmemory-wal-" + + store.getProviderOptions().get(InMemoryWalProvider.PORT_OPTION) + "/wal/" + id, + reference); assertTrue(readerProvider.exists(reference)); writer.getOutputStream().write("header-and-synced-data".getBytes(UTF_8)); @@ -90,7 +91,7 @@ void rejectsInvalidProviderOptionsAndReferences() throws Exception { void rejectsLogDurabilitySyncRequests() throws Exception { try (var store = InMemoryWalStore.start()) { WalProvider provider = newProvider(store.getProviderOptions()); - try (WalProvider.Writer writer = provider.create("tserver:1234", UUID.randomUUID())) { + try (WalProvider.Writer writer = provider.create(UUID.randomUUID())) { assertThrows(IllegalArgumentException.class, () -> writer.sync(Durability.LOG)); } }