diff --git a/doc/Settings.md b/doc/Settings.md index 269a3d029e..6aa29dc770 100644 --- a/doc/Settings.md +++ b/doc/Settings.md @@ -465,3 +465,15 @@ Enables numbered choices for ambiguous single-package `install`, `show`, and `do ``` `--disable-interactivity` and redirected input or output still prevent prompting. + +### deltaIndex + +This feature changes how the client acquires updates to a pre-indexed source. Instead of downloading the whole index every time, it downloads a delta describing only what has changed since a periodically published baseline, and combines the two locally. The baseline is downloaded only when the source does not already have the one the delta was built against. + +The data presented by the source is the same either way; only how it is acquired differs. A source that does not publish a delta is unaffected, and the client falls back to downloading the full index. + +```json + "experimentalFeatures": { + "deltaIndex": true + }, +``` diff --git a/schemas/JSON/settings/settings.schema.0.2.json b/schemas/JSON/settings/settings.schema.0.2.json index e162aaac12..c2271e0838 100644 --- a/schemas/JSON/settings/settings.schema.0.2.json +++ b/schemas/JSON/settings/settings.schema.0.2.json @@ -348,6 +348,11 @@ "description": "Enable support for some commands to resume", "type": "boolean", "default": false + }, + "deltaIndex": { + "description": "Enable acquiring source index updates as a delta against a baseline, rather than downloading the full index", + "type": "boolean", + "default": false } } }, diff --git a/src/AppInstallerCLICore/Command.h b/src/AppInstallerCLICore/Command.h index dcf5aa6b47..c5da3e3f08 100644 --- a/src/AppInstallerCLICore/Command.h +++ b/src/AppInstallerCLICore/Command.h @@ -5,11 +5,11 @@ #include "ExecutionContext.h" #include "Invocation.h" #include "Resources.h" +#include #include #include #include -#include #include #include #include @@ -138,20 +138,6 @@ namespace AppInstaller::CLI std::string m_commandArguments; }; - template - Container InitializeFromMoveOnly(std::initializer_list il) - { - using Value = typename Container::value_type; - Container result; - - for (const auto& v : il) - { - result.emplace_back(std::move(*const_cast(&v))); - } - - return result; - } - void ExecuteWithoutLoggingSuccess(Execution::Context& context, Command* command); int Execute(Execution::Context& context, std::unique_ptr& command); diff --git a/src/AppInstallerCLIE2ETests/Constants.cs b/src/AppInstallerCLIE2ETests/Constants.cs index 5ee25daf69..bd95522928 100644 --- a/src/AppInstallerCLIE2ETests/Constants.cs +++ b/src/AppInstallerCLIE2ETests/Constants.cs @@ -53,6 +53,29 @@ public class Constants public const string RestTestSourceUrl = @"https://localhost:5001/TestKit/TestData/TestRestSource"; public const string RestTestSourceType = "Microsoft.Rest"; + // The delta capable test source. It is published separately from the source above so that a + // client that knows nothing about deltas keeps seeing exactly what it saw before. + public const string DeltaTestSourceName = @"TestDeltaSource"; + public const string DeltaTestSourceDirectory = @"deltaSource"; + public const string DeltaTestSourceUrl = @"https://localhost:5001/TestKit/" + DeltaTestSourceDirectory; + public const string DeltaTestSourceIdentityName = @"WingetE2EDelta.Tests"; + public const string DeltaTestSourceDeltaIdentityName = @"WingetE2EDelta.Tests.Delta"; + public const string DeltaTestSourceBaselineIdentityName = @"WingetE2EDelta.Tests.Baseline"; + public const string DeltaTestSourceIdentifier = DeltaTestSourceIdentityName + "_" + AICLIPackagePublisherHash; + public const string DeltaTestSourceDeltaIdentifier = DeltaTestSourceDeltaIdentityName + "_" + AICLIPackagePublisherHash; + public const string DeltaTestSourceBaselineIdentifier = DeltaTestSourceBaselineIdentityName + "_" + AICLIPackagePublisherHash; + + // The baseline is published older than the full index, because a baseline is captured once + // and then described by a succession of deltas. + public const string DeltaTestSourceVersion = @"2020.805.713.335"; + public const string DeltaTestSourceBaselineVersion = @"2020.805.713.334"; + public const string DeltaTestSourceBaselineRelativePath = @"baseline/source2.msix"; + + // Left out of the baseline, so that it exists only in the delta. Finding it is what tells a + // merged result apart from either half read on its own. + public const string DeltaOnlyManifestFileName = @"TestExampleInstaller.yaml"; + public const string DeltaOnlyPackageIdentifier = @"AppInstallerTest.TestExampleInstaller"; + public const string AICLIPackageFamilyName = "WinGetDevCLI_8wekyb3d8bbwe"; public const string AICLIPackageName = "WinGetDevCLI"; public const string AICLIPackagePublisherHash = "8wekyb3d8bbwe"; diff --git a/src/AppInstallerCLIE2ETests/DeltaSource.cs b/src/AppInstallerCLIE2ETests/DeltaSource.cs new file mode 100644 index 0000000000..db2814a9cf --- /dev/null +++ b/src/AppInstallerCLIE2ETests/DeltaSource.cs @@ -0,0 +1,247 @@ +// ----------------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. +// +// ----------------------------------------------------------------------------- + +namespace AppInstallerCLIE2ETests +{ + using AppInstallerCLIE2ETests.Helpers; + using NUnit.Framework; + + /// + /// Tests the delta index against the deployed package mechanism. + /// + /// Unit tests cover the shape of a delta source and the merging of a delta with its baseline, + /// but they run unpackaged, so they can only ever reach the local file store. Everything here + /// is about the half they structurally cannot touch: deploying the packages, keeping three + /// separate deployed identities apart, and the clean up that follows a removal. + /// + public class DeltaSource : BaseCommand + { + /// + /// The experimental feature, as it is named in the settings file. + /// + private const string DeltaFeatureName = "deltaIndex"; + + /// + /// A package that is in the baseline, and so is not one the delta has to supply. + /// + private const string BaselinePackageIdentifier = Constants.PortableExePackageId; + + /// + /// One time set up. + /// + [OneTimeSetUp] + public void OneTimeSetup() + { + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true); + } + + /// + /// One time tear down. + /// + [OneTimeTearDown] + public void OneTimeTeardown() + { + RemoveDeltaSource(); + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false); + } + + /// + /// Test set up. + /// + [SetUp] + public void Setup() + { + RemoveDeltaSource(); + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true); + } + + /// + /// Adding a delta capable source acquires the delta and the baseline rather than the full + /// index, and what it then presents is the two of them merged. + /// + [Test] + public void DeltaSourceAdd() + { + AddDeltaSource(); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True, "The delta package should be deployed"); + Assert.That( + GetDeployedPackageVersion(Constants.DeltaTestSourceBaselineIdentityName), + Is.EqualTo(Constants.DeltaTestSourceBaselineVersion), + "The baseline should have been deployed at its own version"); + + // The baseline is a package of its own, so a source on the delta holds no full index at + // all. Were the two to share an identity this could not be asked. + Assert.That( + IsPackageDeployed(Constants.DeltaTestSourceIdentityName), + Is.False, + "The full index should not have been acquired"); + + // This package was left out of the baseline, so it exists only in the delta. + RequireFound(Constants.DeltaOnlyPackageIdentifier); + + // And a package that is in the baseline is still there, so the delta did not displace it. + RequireFound(BaselinePackageIdentifier); + } + + /// + /// A source added while the feature was off holds the full index, and goes on holding it. + /// + /// Which packages a source is made of is established when it is added and recorded in its + /// details; an update reads those details and never probes for anything else. So turning + /// the feature on does not move an existing source onto the delta, and the source has to be + /// re-added for that. This is deliberate -- see the no-migration decision -- and it is + /// asserted here because nothing about it is obvious from the outside. + /// + [Test] + public void DeltaSourceNotAdoptedByExistingFullIndexSource() + { + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false); + + AddDeltaSource(); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "No delta should be involved while the feature is off"); + Assert.That( + GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), + Is.EqualTo(Constants.DeltaTestSourceVersion), + "The full index should be deployed at its own version"); + + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true); + + var updateResult = TestCommon.RunAICLICommand("source update", $"-n {Constants.DeltaTestSourceName}"); + Assert.That(updateResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); + Assert.That(updateResult.StdOut, Does.Contain("Done")); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "The update should not have acquired a delta"); + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceBaselineIdentityName), Is.False, "The update should not have acquired a baseline"); + Assert.That( + GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), + Is.EqualTo(Constants.DeltaTestSourceVersion), + "The source should still be served by the full index"); + + // The full index holds everything, so what the source presents is unchanged. + RequireFound(Constants.DeltaOnlyPackageIdentifier); + RequireFound(BaselinePackageIdentifier); + } + + /// + /// Disabling delta after adding a source acquires the full index and keeps it usable. + /// + /// Whether to explicitly update before searching. + [TestCase(false)] + [TestCase(true)] + public void DeltaSourceFeatureDisabled(bool updateSource) + { + AddDeltaSource(); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True); + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceBaselineIdentityName), Is.True); + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceIdentityName), Is.False); + RequireFound(Constants.DeltaOnlyPackageIdentifier); + RequireFound(BaselinePackageIdentifier); + + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false); + + if (updateSource) + { + var updateResult = TestCommon.RunAICLICommand("source update", $"-n {Constants.DeltaTestSourceName}"); + Assert.That(updateResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); + Assert.That(updateResult.StdOut, Does.Contain("Done")); + } + + RequireFound(Constants.DeltaOnlyPackageIdentifier); + RequireFound(BaselinePackageIdentifier); + Assert.That( + GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), + Is.EqualTo(Constants.DeltaTestSourceVersion), + "The source should be served by the full index with delta disabled"); + + var removeResult = TestCommon.RunAICLICommand("source remove", $"-n {Constants.DeltaTestSourceName}"); + Assert.That(removeResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceIdentityName), Is.False); + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False); + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceBaselineIdentityName), Is.False); + } + + /// + /// Removing the source takes both deployed packages with it. A delta source leaves more + /// behind than an ordinary one, so there is more to get wrong. + /// + [Test] + public void DeltaSourceRemove() + { + AddDeltaSource(); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True); + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceBaselineIdentityName), Is.True); + + var removeResult = TestCommon.RunAICLICommand("source remove", $"-n {Constants.DeltaTestSourceName}"); + Assert.That(removeResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); + Assert.That(removeResult.StdOut, Does.Contain("Done")); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "The delta package should have been removed"); + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceBaselineIdentityName), Is.False, "The baseline package should have been removed"); + + var listResult = TestCommon.RunAICLICommand("source list", $"-n {Constants.DeltaTestSourceName}"); + Assert.That(listResult.ExitCode, Is.EqualTo(Constants.ErrorCode.ERROR_SOURCE_NAME_DOES_NOT_EXIST)); + } + + /// + /// Adds the delta source, requiring that it succeed. + /// + private static void AddDeltaSource() + { + var result = TestCommon.RunAICLICommand("source add", $"{Constants.DeltaTestSourceName} {Constants.DeltaTestSourceUrl} --trust-level trusted"); + Assert.That(result.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); + Assert.That(result.StdOut, Does.Contain("Done")); + } + + /// + /// Removes the delta source, tolerating its not having been added. + /// + private static void RemoveDeltaSource() + { + TestCommon.RunAICLICommand("source remove", $"-n {Constants.DeltaTestSourceName}"); + } + + /// + /// Requires that the delta source presents the given package. + /// + /// The package identifier. + private static void RequireFound(string packageIdentifier) + { + var result = TestCommon.RunAICLICommand("search", $"{packageIdentifier} --source {Constants.DeltaTestSourceName}"); + Assert.That(result.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK), $"{packageIdentifier} should be found"); + Assert.That(result.StdOut, Does.Contain(packageIdentifier)); + } + + /// + /// Whether a package with the given identity name is deployed for the current user. + /// + /// The package identity name. + /// True if it is deployed. + private static bool IsPackageDeployed(string identityName) + { + return !string.IsNullOrEmpty(GetDeployedPackageVersion(identityName)); + } + + /// + /// Gets the version of a deployed package, or an empty string when it is not deployed. + /// + /// The package identity name. + /// The version, or an empty string. + private static string GetDeployedPackageVersion(string identityName) + { + // The name has to match exactly, which matters here because the delta's identity name + // has the full index's as a prefix. + var result = TestCommon.RunCommandWithResult( + "powershell", + $"-Command \"(Get-AppxPackage | Where-Object {{ $_.Name -ceq '{identityName}' }} | Select-Object -First 1).Version\""); + + return result.StdOut.Trim(); + } + } +} diff --git a/src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs b/src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs index 8468725bd2..b3f08d918d 100644 --- a/src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs +++ b/src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs @@ -192,6 +192,20 @@ public static void GenerateE2ESource() { CertFile = testParams.PackageCertificatePath, }, + Delta = new () + { + RelativeDirectory = Constants.DeltaTestSourceDirectory, + IdentityName = Constants.DeltaTestSourceIdentityName, + DeltaIdentityName = Constants.DeltaTestSourceDeltaIdentityName, + BaselineIdentityName = Constants.DeltaTestSourceBaselineIdentityName, + Version = Constants.DeltaTestSourceVersion, + BaselineVersion = Constants.DeltaTestSourceBaselineVersion, + BaselineRelativePath = Constants.DeltaTestSourceBaselineRelativePath, + ManifestsNotInBaseline = new () + { + Constants.DeltaOnlyManifestFileName, + }, + }, }; WinGetLocalSource.CreateLocalSource(e2eSource); diff --git a/src/AppInstallerCLIE2ETests/TestData/localsource.json b/src/AppInstallerCLIE2ETests/TestData/localsource.json index 24ac9aebc8..2a4fad140f 100644 --- a/src/AppInstallerCLIE2ETests/TestData/localsource.json +++ b/src/AppInstallerCLIE2ETests/TestData/localsource.json @@ -53,5 +53,17 @@ "Signature": { "CertFile": "%TestSigningCert_PfxPath%", "Password": "%TestSigningCert_Password%" + }, + "Delta": { + "RelativeDirectory": "deltaSource", + "IdentityName": "WingetE2EDelta.Tests", + "DeltaIdentityName": "WingetE2EDelta.Tests.Delta", + "BaselineIdentityName": "WingetE2EDelta.Tests.Baseline", + "Version": "2020.805.713.335", + "BaselineVersion": "2020.805.713.334", + "BaselineRelativePath": "baseline/source2.msix", + "ManifestsNotInBaseline": [ + "TestExampleInstaller.yaml" + ] } } diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj index 4051f4c994..8e4d7233e1 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj @@ -218,6 +218,7 @@ + @@ -338,6 +339,7 @@ + diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters index f29961ab30..5a7a05e52e 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters @@ -81,6 +81,9 @@ Header Files + + Header Files + Header Files @@ -116,6 +119,9 @@ Source Files + + Source Files + Source Files @@ -425,15 +431,15 @@ Source Files\Repository - - Source Files - Source Files\Repository Source Files\Repository + + Source Files\CLI + diff --git a/src/AppInstallerCLITests/ExperimentalFeature.cpp b/src/AppInstallerCLITests/ExperimentalFeature.cpp index 6c75f6040b..41ebf035b2 100644 --- a/src/AppInstallerCLITests/ExperimentalFeature.cpp +++ b/src/AppInstallerCLITests/ExperimentalFeature.cpp @@ -28,6 +28,48 @@ TEST_CASE("ExperimentalFeature None", "[experimentalFeature]") REQUIRE(ExperimentalFeature::IsEnabled(ExperimentalFeature::Feature::None)); } +// Registering a feature takes four separate edits in three files, and nothing connects them. A +// flag added to the enum without a GetFeature case makes `winget features` throw rather than +// merely omit the feature, and one without a setting throws from IsEnabled. Both are caught here +// for every feature at once, so the cost of the split registration is paid by this test rather +// than by whoever adds the next one. +TEST_CASE("ExperimentalFeature AllFeaturesAreRegistered", "[experimentalFeature]") +{ + auto again = DeleteUserSettingsFiles(); + + std::set jsonNames; + + for (const auto& feature : ExperimentalFeature::GetAllFeatures()) + { + INFO(feature.Name()); + + REQUIRE(!feature.Name().empty()); + REQUIRE(!feature.JsonName().empty()); + + // The json name is what a user types, so two features cannot share one. + REQUIRE(jsonNames.insert(feature.JsonName()).second); + + // An experimental feature is off until it is asked for. The empty settings are written + // explicitly so that this does not depend on what a previous iteration left behind. + { + SetSetting(Stream::PrimaryUserSettings, "{}"); + UserSettingsTest userSettingTest; + + REQUIRE_FALSE(ExperimentalFeature::IsEnabled(feature.GetFeature(), userSettingTest)); + } + + // The name that `winget features` prints has to be the one that actually enables the + // feature, which is the only thing tying the enum value to its setting. + { + std::string json = R"({ "experimentalFeatures": { ")" + std::string{ feature.JsonName() } + R"(": true } })"; + SetSetting(Stream::PrimaryUserSettings, json); + UserSettingsTest userSettingTest; + + REQUIRE(ExperimentalFeature::IsEnabled(feature.GetFeature(), userSettingTest)); + } + } +} + TEST_CASE("ExperimentalFeature ExperimentalCmd", "[experimentalFeature]") { auto again = DeleteUserSettingsFiles(); @@ -104,4 +146,4 @@ TEST_CASE("ExperimentalFeature InteractivePackageSelection", "[experimentalFeatu { return item.GetFeature() == ExperimentalFeature::Feature::InteractivePackageSelection; })); -} \ No newline at end of file +} diff --git a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp index fe36dd1b31..6678967a63 100644 --- a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp +++ b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp @@ -3,11 +3,17 @@ #include "pch.h" #include "TestSource.h" #include "TestCommon.h" +#include "TestHooks.h" +#include "TestMsixPackage.h" #include "TestSettings.h" +#include "SQLiteIndexTestCommon.h" #include #include #include #include +#include +#include +#include #include using namespace std::string_literals; @@ -226,3 +232,1188 @@ TEST_CASE("PIPS_Remove", "[pips]") UninstallCertFromSignedPackage(index); } } + +using namespace AppInstaller::Repository::Microsoft::PreIndexed; + +namespace +{ + // The identity that the built-in source has always been known by. + constexpr std::string_view s_WinGetSourceIdentity = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; + + // Stand in identities for the data format cases, which care only that a value survives. + constexpr std::string_view s_FullIdentity = "Base_8wekyb3d8bbwe"sv; + constexpr std::string_view s_DeltaIdentity = "Delta_8wekyb3d8bbwe"sv; + constexpr std::string_view s_BaselineIdentity = "Baseline_8wekyb3d8bbwe"sv; +} + +TEST_CASE("PreIndexedSourceData_BareIdentity", "[pips][source_data]") +{ + SourceData data{ s_WinGetSourceIdentity }; + + REQUIRE(data.FullIdentity() == s_WinGetSourceIdentity); + REQUIRE(!data.HasDeltaIdentity()); + REQUIRE(!data.HasBaselineIdentity()); + REQUIRE(data.SourceIdentity() == s_WinGetSourceIdentity); + + // A source with no delta round trips to exactly what it was read from, so that the stored + // value does not churn for sources that are not delta capable. + REQUIRE(data.Serialize() == s_WinGetSourceIdentity); +} + +TEST_CASE("PreIndexedSourceData_Empty", "[pips][source_data]") +{ + SourceData data{ ""sv }; + + REQUIRE(!data.HasFullIdentity()); + REQUIRE(!data.HasDeltaIdentity()); + REQUIRE(!data.HasBaselineIdentity()); + REQUIRE(data.Serialize().empty()); +} + +TEST_CASE("PreIndexedSourceData_RoundTrip", "[pips][source_data]") +{ + SourceData original; + original.FullIdentity(std::string{ s_FullIdentity }); + original.DeltaIdentity(std::string{ s_DeltaIdentity }); + original.BaselineIdentity(std::string{ s_BaselineIdentity }); + + std::string serialized = original.Serialize(); + INFO(serialized); + + SourceData parsed{ serialized }; + + REQUIRE(parsed.FullIdentity() == s_FullIdentity); + REQUIRE(parsed.DeltaIdentity() == s_DeltaIdentity); + REQUIRE(parsed.BaselineIdentity() == s_BaselineIdentity); + REQUIRE(parsed.Serialize() == serialized); +} + +TEST_CASE("PreIndexedSourceData_DeltaPairRoundTrip", "[pips][source_data]") +{ + // Older delta source data can name only the delta pair, without a full index to fall back to. + SourceData original; + original.DeltaIdentity(std::string{ s_DeltaIdentity }); + original.BaselineIdentity(std::string{ s_BaselineIdentity }); + + std::string serialized = original.Serialize(); + INFO(serialized); + + SourceData parsed{ serialized }; + + REQUIRE(!parsed.HasFullIdentity()); + REQUIRE(parsed.DeltaIdentity() == s_DeltaIdentity); + REQUIRE(parsed.BaselineIdentity() == s_BaselineIdentity); + + // With no full index to be known by, the source is keyed on its delta. + REQUIRE(parsed.SourceIdentity() == s_DeltaIdentity); +} + +TEST_CASE("PreIndexedSourceData_SourceIdentityPrefersTheFullIndex", "[pips][source_data]") +{ + // The identity a source is known by must not depend on which of its packages it happens to + // be using, since its local state directory and cross process lock are derived from it. + SourceData data; + data.FullIdentity(std::string{ s_FullIdentity }); + data.DeltaIdentity(std::string{ s_DeltaIdentity }); + data.BaselineIdentity(std::string{ s_BaselineIdentity }); + + REQUIRE(data.SourceIdentity() == s_FullIdentity); +} + +TEST_CASE("PreIndexedSourceData_UnknownMembersIgnored", "[pips][source_data]") +{ + // A newer client may add to this value; an older one must still be able to use the source. + std::string withUnknown = R"({"fullIdentity":")" + std::string{ s_FullIdentity } + R"(","somethingNew":42})"; + + SourceData data{ withUnknown }; + + REQUIRE(data.FullIdentity() == s_FullIdentity); + REQUIRE(!data.HasDeltaIdentity()); +} + +TEST_CASE("PreIndexedSourceData_Malformed", "[pips][source_data]") +{ + // A value that is structured but unreadable is an error, not a package family name. + std::string truncated = R"({"fullIdentity":)"; + std::string noIdentity = R"({"baselineIdentity":")" + std::string{ s_BaselineIdentity } + R"("})"; + std::string wrongType = R"({"fullIdentity":42})"; + + REQUIRE_THROWS(SourceData{ truncated }); + REQUIRE_THROWS(SourceData{ noIdentity }); + REQUIRE_THROWS(SourceData{ wrongType }); +} + +TEST_CASE("PreIndexedSourceData_WellKnownWinGetSourceNamesDelta", "[pips][source_data]") +{ + SourceData data{ GetWellKnownSourceDetails(WellKnownSource::WinGet).Data }; + + REQUIRE(data.FullIdentity() == s_WinGetSourceIdentity); + REQUIRE(data.HasDeltaIdentity()); + REQUIRE(data.HasBaselineIdentity()); + + // Each package has an identity of its own, so that a store can hold them all at once. + REQUIRE(data.BaselineIdentity() != data.FullIdentity()); + REQUIRE(data.BaselineIdentity() != data.DeltaIdentity()); + + // The source's own identity has to stay the one it has always been known by, since the local + // state directory and the cross process lock are derived from it. + REQUIRE(GetWellKnownSourceDetails(WellKnownSource::WinGet).Identifier == data.FullIdentity()); + REQUIRE(data.SourceIdentity() == data.FullIdentity()); +} + +// The cases below build their own packages rather than using the signed test data, so they need +// no certificate installation and therefore no administrator rights. They exercise the local file +// store, which is the only mechanism an unpackaged test process can reach; the deployed mechanism +// is covered by E2E tests. + +namespace +{ + using SQLiteIndex = AppInstaller::Repository::Microsoft::SQLiteIndex; + using SQLiteStorageBase = AppInstaller::SQLite::SQLiteStorageBase; + + // Where the source publishes its baseline, and the version it publishes there. The delta + // records both, and the client checks the second before downloading anything. + constexpr std::string_view s_BaselineRelativePath = "baselines/current/baseline.msix"sv; + constexpr std::string_view s_BaselineVersion = "1.2.3.4"sv; + + // The next baseline lineage. The service chooses where its baselines live, so a new one is + // published beside the old rather than over it, and the two overlap for a time. + constexpr std::string_view s_NextBaselineRelativePath = "baselines/next/baseline.msix"sv; + constexpr std::string_view s_NextBaselineVersion = "5.6.7.8"sv; + + constexpr std::string_view s_DeltaMsixName = "delta.msix"sv; + constexpr std::string_view s_BaselineMsixName = "baseline.msix"sv; + + // Where the source publishes its full index. The client prefers this name over the one it + // falls back to, so this is the one a test publishes to and reads back. + constexpr std::string_view s_RemoteIndexMsixName = "source2.msix"sv; + + // The versions that the source publishes its full index and delta at, in the order that the + // cases publish them. + constexpr std::string_view s_FirstVersion = "1.0.0.0"sv; + constexpr std::string_view s_SecondVersion = "2.0.0.0"sv; + constexpr std::string_view s_ThirdVersion = "3.0.0.0"sv; + constexpr std::string_view s_FourthVersion = "4.0.0.0"sv; + + // A baseline version that no delta names, used to stand in for a baseline that has moved on. + constexpr std::string_view s_UnexpectedBaselineVersion = "9.9.9.9"sv; + + // The packages that the cases publish. Each identifier is written into an index and then read + // back out of the source, so it is named rather than repeated at both ends. + const std::string s_Package1Id = "Publisher1.Id"; + const std::string s_Package2Id = "Publisher2.Id"; + const std::string s_Package3Id = "Publisher3.Id"; + const std::string s_Package4Id = "Publisher4.Id"; + const std::string s_Package5Id = "Publisher5.Id"; + + IndexFields MakeIndexFields(const std::string& id) + { + return IndexFields{ + id, + id + " Name", + "moniker"s, + "1.0"s, + ""s, + { "tag1" }, + { "command1" }, + id + "/1.0" }; + } + + // Where the local file store keeps a source's packages. + fs::path GetStatePathFor(const std::string& sourceIdentity) + { + fs::path result = GetPathTo(Runtime::PathName::LocalState); + result /= AppInstaller::Repository::Microsoft::PreIndexedPackageSourceFactory::Type(); + result /= sourceIdentity; + return result; + } + + // A deployed package is held outside of our local state entirely, so the store that stands in + // for that mechanism holds its packages under an identity of its own. That keeps the two as + // separate as the real ones are. + constexpr std::string_view s_DeployedIdentitySuffix = ".Deployed"sv; + + // Where the stand in for the deployed store keeps a source's packages. + fs::path GetDeployedStatePathFor(const std::string& sourceIdentity) + { + return GetStatePathFor(sourceIdentity + std::string{ s_DeployedIdentitySuffix }); + } + + void CleanSourcesFor(const std::string& sourceIdentity) + { + RemoveSetting(Stream::UserSources); + RemoveSetting(Stream::SourcesMetadata); + fs::remove_all(GetStatePathFor(sourceIdentity)); + fs::remove_all(GetDeployedStatePathFor(sourceIdentity)); + } + + // AddSource takes the details by const reference, so what a source actually recorded has to be + // read back out of the source list. + SourceDetails GetStoredDetails(std::string_view name) + { + for (const auto& details : GetSources()) + { + if (details.Name == name) + { + return details; + } + } + + FAIL("Source was not found in the current sources"); + return {}; + } + + // A baseline that the source publishes, and the location and version that a delta computed + // against it names. + // + // The service chooses where its baselines live, so there can be more than one at a time; a + // newly published lineage overlaps the one it is replacing rather than overwriting it. + struct PublishedBaseline + { + PublishedBaseline(std::string_view relativePath, std::string_view version) : + RelativePath(relativePath), Version(version) {} + + // The designated baseline, and the empty delta that designating it produces. Nothing + // reads the latter; it exists because designation and delta generation are one prepare. + TempFile IndexFile{ "pips_baseline"s, ".db"s }; + TempFile DesignationDelta{ "pips_baseline_delta"s, ".db"s }; + + std::string RelativePath; + std::string Version; + }; + + // A pre-indexed source published into a local directory. + // + // The packages are built here rather than checked in, so that a test can choose the identities + // and versions that the delta pairing logic turns on. They are unsigned, so every test using + // this must hold a TestHook::SetSourcePackageTrustValidation_Override. + struct TestPreIndexedSource + { + TempDirectory Remote{ "pips_remote" }; + + // The long lived index that the source is built from. It is never prepared itself; each + // publish prepares a copy, since preparing is a one way operation. + TempFile WorkingFile{ "pips_working"s, ".db"s }; + + // The baseline that the source's deltas are computed against, and the lineage that the + // service moves to when it rolls the baseline. + PublishedBaseline Baseline{ s_BaselineRelativePath, s_BaselineVersion }; + PublishedBaseline NextBaseline{ s_NextBaselineRelativePath, s_NextBaselineVersion }; + + // Every package the source publishes has an identity of its own, since the client has to + // be able to ask for one without getting another, and a store that holds packages by + // identity could not hold two that shared one. + std::string FullIdentityName = "WinGetTestSourceIndex"; + std::string DeltaIdentityName = "WinGetTestSourceIndexDelta"; + std::string BaselineIdentityName = "WinGetTestSourceIndexBaseline"; + + std::string FullIndexFamilyName; + std::string DeltaFamilyName; + std::string BaselineFamilyName; + + TestPreIndexedSource(const std::vector& packages) + { + SQLiteIndex index = SQLiteIndex::CreateNew(WorkingFile.GetPath().u8string(), SQLiteVersion{ 2, 1 }); + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, "0"); + + for (const auto& fields : packages) + { + auto manifest = CreateManifest(fields); + index.AddManifest(manifest, fields.Path); + } + } + + void AddPackage(const IndexFields& fields) + { + SQLiteIndex index = SQLiteIndex::Open(WorkingFile.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + + if (!m_baseTimeReset) + { + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, ""); + m_baseTimeReset = true; + } + + auto manifest = CreateManifest(fields); + index.AddManifest(manifest, fields.Path); + } + + // Prepares a copy of the working index, designating it as a baseline in the same prepare, + // and publishes it where the delta will say it is. Everything the working index does after + // this is what a delta against that baseline will describe. + void PublishBaseline() + { + PublishBaseline(Baseline); + } + + void PublishBaseline(PublishedBaseline& baseline) + { + fs::copy_file(WorkingFile.GetPath(), baseline.IndexFile.GetPath(), fs::copy_options::overwrite_existing); + + { + SQLiteIndex index = SQLiteIndex::Open(baseline.IndexFile.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + + index.SetProperty(SQLiteIndex::Property::DeltaMarkAsBaseline, "true"); + index.SetProperty(SQLiteIndex::Property::DeltaOutputPath, baseline.DesignationDelta.GetPath().u8string()); + index.SetProperty(SQLiteIndex::Property::DeltaBaselineRelativeSourcePath, baseline.RelativePath); + index.SetProperty(SQLiteIndex::Property::DeltaBaselinePackageVersion, baseline.Version); + + index.PrepareForPackaging(); + } + + BaselineFamilyName = PublishPackage(baseline.IndexFile.GetPath(), baseline.RelativePath, BaselineIdentityName, baseline.Version); + } + + // Prepares a copy of the working index and publishes it as the source's full index, + // optionally publishing the delta that preparing produces alongside it. + // + // The delta is published at the same version as the full index, since the two describe the + // same contents. Nothing in the client requires that, but there is no reason to differ. + void Publish(std::string_view version, bool publishDelta = false) + { + Publish(Baseline, version, publishDelta); + } + + void Publish(const PublishedBaseline& baseline, std::string_view version, bool publishDelta) + { + TempFile prepared{ "pips_prepared"s, ".db"s }; + TempFile delta{ "pips_delta"s, ".db"s }; + + fs::copy_file(WorkingFile.GetPath(), prepared.GetPath(), fs::copy_options::overwrite_existing); + + { + SQLiteIndex index = SQLiteIndex::Open(prepared.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + + if (publishDelta) + { + index.SetProperty(SQLiteIndex::Property::DeltaBaselineIndexPath, baseline.IndexFile.GetPath().u8string()); + index.SetProperty(SQLiteIndex::Property::DeltaOutputPath, delta.GetPath().u8string()); + index.SetProperty(SQLiteIndex::Property::DeltaBaselineRelativeSourcePath, baseline.RelativePath); + index.SetProperty(SQLiteIndex::Property::DeltaBaselinePackageVersion, baseline.Version); + } + + index.PrepareForPackaging(); + } + + FullIndexFamilyName = PublishPackage(prepared.GetPath(), s_RemoteIndexMsixName, FullIdentityName, std::string{ version }); + + if (publishDelta) + { + DeltaFamilyName = PublishPackage(delta.GetPath(), s_DeltaMsixName, DeltaIdentityName, std::string{ version }); + } + } + + // Replaces the published baseline with one at a version the delta does not name. + void RepublishBaselineAtVersion(std::string_view version) + { + PublishPackage(Baseline.IndexFile.GetPath(), Baseline.RelativePath, BaselineIdentityName, std::string{ version }); + } + + // Where a published baseline package sits on the "remote". + fs::path RemotePathFor(const PublishedBaseline& baseline) const + { + return Remote.GetPath() / baseline.RelativePath; + } + + SourceDetails MakeDetails(std::string name = "TestName") const + { + SourceDetails details; + details.Name = std::move(name); + details.Type = AppInstaller::Repository::Microsoft::PreIndexedPackageSourceFactory::Type(); + details.Arg = Remote.GetPath().u8string(); + return details; + } + + fs::path StatePath() const + { + REQUIRE(!FullIndexFamilyName.empty()); + return GetStatePathFor(FullIndexFamilyName); + } + + // Clears every trace of this source, whichever form it settles on. + void CleanSources() const + { + if (!FullIndexFamilyName.empty()) + { + CleanSourcesFor(FullIndexFamilyName); + } + + if (!DeltaFamilyName.empty()) + { + CleanSourcesFor(DeltaFamilyName); + } + } + + // Where the stand in for the deployed store holds this source's packages. + fs::path DeployedStatePath() const + { + REQUIRE(!FullIndexFamilyName.empty()); + return GetDeployedStatePathFor(FullIndexFamilyName); + } + + private: + std::string PublishPackage( + const fs::path& indexFile, + std::string_view relativePath, + const std::string& identityName, + const std::string& version) + { + fs::path target = Remote.GetPath() / std::string{ relativePath }; + fs::create_directories(target.parent_path()); + fs::remove(target); + + TestIndexPackageDefinition definition; + definition.Name = identityName; + definition.Version = version; + + return CreateTestIndexPackage(target, indexFile, definition); + } + + bool m_baseTimeReset = false; + }; + + // The identifiers the named source presents, read back through the source itself rather than + // out of the files, so that what is asserted is what a user would see. + // + // A background update interval may be supplied to let the open itself decide that an update is + // due. It has to be non zero, since zero means that the source is never updated automatically. + std::set GetSourcePackageIds(std::string_view sourceName, std::optional backgroundUpdateInterval = {}) + { + Source source{ sourceName }; + + if (backgroundUpdateInterval) + { + source.SetBackgroundUpdateInterval(backgroundUpdateInterval.value()); + } + + TestProgress callback; + source.Open(callback); + + std::set result; + + for (const auto& match : source.Search({}).Matches) + { + result.insert(match.Package->GetProperty(PackageProperty::Id).get()); + } + + return result; + } + + // Stands in for the deployed package store. + // + // Packages are held as files, as the local file store does, but under an identity of their + // own so that the two mechanisms are as separate as the real ones are. Reads are reported as + // allowed without the lock, which is what the deployed store reports and what the composite + // store has to carry through correctly. + struct TestDeployedPackageStore : public IPackageStore + { + TestDeployedPackageStore(const SourceDetails& details) + { + std::string identity = details.Identifier.empty() ? SourceData{ details.Data }.SourceIdentity() : details.Identifier; + REQUIRE(!identity.empty()); + + SourceDetails deployedDetails = details; + deployedDetails.Identifier = identity + std::string{ s_DeployedIdentitySuffix }; + + m_inner = CreateLocalFilePackageStore(deployedDetails); + } + + std::optional Acquire(const PackageKey& package, const std::string& location, IProgressCallback& progress) override + { + return m_inner->Acquire(package, location, progress); + } + + void Persist(AcquiredPackage&& package, IProgressCallback& progress) override + { + m_inner->Persist(std::move(package), progress); + } + + std::optional GetVersion(const PackageKey& package) const override + { + return m_inner->GetVersion(package); + } + + std::optional GetIndex(const PackageKey& package, IProgressCallback& progress) override + { + return m_inner->GetIndex(package, progress); + } + + void Remove(const std::vector& packages, IProgressCallback& progress) override + { + m_inner->Remove(packages, progress); + } + + void RemovePackage(const PackageKey& package, IProgressCallback& progress) override + { + m_inner->RemovePackage(package, progress); + } + + Synchronization::CrossProcessLock Lock(IProgressCallback& progress, bool isBackground = false) override + { + return m_inner->Lock(progress, isBackground); + } + + bool AllowsUnlockedRead() const override { return true; } + + private: + std::unique_ptr m_inner; + }; + + std::unique_ptr CreateTestDeployedPackageStore(const SourceDetails& details) + { + return std::make_unique(details); + } +} + +TEST_CASE("PIPS_LocalFile_FullIndex", "[pips][local_file]") +{ + // Held off explicitly, so that this guards the behavior of a source that is not delta capable + // rather than whatever the machine running the test happens to have enabled. + TestHook::SetSingleExperimentalFeature_Override deltaDisabled{ ExperimentalFeature::Feature::DeltaIndex, false }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id), MakeIndexFields(s_Package2Id) } }; + source.Publish(s_FirstVersion); + + source.CleanSources(); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_IndexMsixName)); + + // The delta's slots are untouched by a source that is not using one. + REQUIRE(!fs::exists(state / s_DeltaMsixName)); + REQUIRE(!fs::exists(state / s_BaselineMsixName)); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id }); + + // A newer full index is acquired and replaces what is held. + source.AddPackage(MakeIndexFields(s_Package3Id)); + source.Publish(s_SecondVersion); + + bool progressCalled = false; + callback.m_OnProgress = [&](uint64_t, uint64_t, ProgressType) { progressCalled = true; }; + + REQUIRE(UpdateSource(details.Name, callback)); + REQUIRE(progressCalled); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); + + REQUIRE(RemoveSource(details.Name, callback)); + REQUIRE(!fs::exists(state)); +} + +TEST_CASE("PIPS_LocalFile_Delta_Add", "[pips][local_file][delta]") +{ + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id), MakeIndexFields(s_Package2Id) } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields(s_Package3Id)); + source.Publish(s_SecondVersion, true); + + source.CleanSources(); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + SourceDetails stored = GetStoredDetails(details.Name); + + // Record every identity while adding, including the full index that can serve the source + // when delta is disabled, without acquiring the full index. + SourceData data{ stored.Data }; + REQUIRE(data.FullIdentity() == source.FullIndexFamilyName); + REQUIRE(data.DeltaIdentity() == source.DeltaFamilyName); + REQUIRE(data.BaselineIdentity() == source.BaselineFamilyName); + + // The baseline is a package in its own right, not the full index under another name. + REQUIRE(source.BaselineFamilyName != source.FullIndexFamilyName); + + // Both forms use the same source identity, so disabling delta preserves its state and lock. + REQUIRE(stored.Identifier == source.FullIndexFamilyName); + + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_DeltaMsixName)); + REQUIRE(fs::exists(state / s_BaselineMsixName)); + + // Nothing acquired the full index, which is the whole point. + REQUIRE(!fs::exists(state / s_IndexMsixName)); + + // The package added after the baseline was captured is only in the delta, so finding it proves + // that the two were merged rather than that either was read on its own. + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); +} + +TEST_CASE("PIPS_LocalFile_Delta_FeatureDisabledUsesFullIndex", "[pips][local_file][delta]") +{ + bool explicitUpdate = GENERATE(false, true); + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; + source.PublishBaseline(); + source.AddPackage(MakeIndexFields(s_Package2Id)); + source.Publish(s_SecondVersion, true); + source.CleanSources(); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + { + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + REQUIRE(AddSource(details, callback)); + } + + SourceDetails stored = GetStoredDetails(details.Name); + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_DeltaMsixName)); + REQUIRE(fs::exists(state / s_BaselineMsixName)); + REQUIRE(!fs::exists(state / s_IndexMsixName)); + + TestHook::SetSingleExperimentalFeature_Override deltaDisabled{ ExperimentalFeature::Feature::DeltaIndex, false }; + if (explicitUpdate) + { + REQUIRE(UpdateSource(details.Name, callback)); + } + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id }); + REQUIRE(fs::exists(state / s_IndexMsixName)); + REQUIRE(GetStoredDetails(details.Name).Data == stored.Data); + REQUIRE(GetStoredDetails(details.Name).Identifier == stored.Identifier); + + REQUIRE(RemoveSource(details.Name, callback)); + REQUIRE(!fs::exists(state)); +} + +TEST_CASE("PIPS_LocalFile_Delta_UpdateKeepsBaseline", "[pips][local_file][delta]") +{ + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields(s_Package2Id)); + source.Publish(s_SecondVersion, true); + + source.CleanSources(); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + fs::path baselinePackage = source.StatePath() / s_BaselineMsixName; + REQUIRE(fs::exists(baselinePackage)); + auto baselineWriteTime = fs::last_write_time(baselinePackage); + + // A newer delta against the same baseline. This is the common case, and the saving it buys is + // that no baseline traffic occurs at all. + source.AddPackage(MakeIndexFields(s_Package3Id)); + source.Publish(s_ThirdVersion, true); + + REQUIRE(UpdateSource(details.Name, callback)); + + REQUIRE(fs::last_write_time(baselinePackage) == baselineWriteTime); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); +} + +TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenNoDeltaPublished", "[pips][local_file][delta]") +{ + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; + source.Publish(s_FirstVersion); + + source.CleanSources(); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + // A source that publishes no delta is an ordinary answer rather than a failure, so the added + // source records no delta identity and holds the full index. + SourceData data{ GetStoredDetails(details.Name).Data }; + REQUIRE(data.FullIdentity() == source.FullIndexFamilyName); + REQUIRE(!data.HasDeltaIdentity()); + + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_IndexMsixName)); + REQUIRE(!fs::exists(state / s_DeltaMsixName)); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id }); +} + +TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenBaselineVersionDiffers", "[pips][local_file][delta]") +{ + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields(s_Package2Id)); + source.Publish(s_SecondVersion, true); + + // Whatever is at the location the delta names is not the baseline it was computed against, and + // the version header says so before anything is downloaded. + source.RepublishBaselineAtVersion(s_UnexpectedBaselineVersion); + + source.CleanSources(); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + // The delta cannot serve the source, so the full index does. + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_IndexMsixName)); + REQUIRE(!fs::exists(state / s_BaselineMsixName)); + + SourceData data{ GetStoredDetails(details.Name).Data }; + REQUIRE(!data.HasDeltaIdentity()); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id }); +} + +TEST_CASE("PIPS_LocalFile_Delta_RemoveClearsEverySlot", "[pips][local_file][delta]") +{ + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields(s_Package2Id)); + source.Publish(s_SecondVersion, true); + + source.CleanSources(); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_DeltaMsixName)); + REQUIRE(fs::exists(state / s_BaselineMsixName)); + + REQUIRE(RemoveSource(details.Name, callback)); + + REQUIRE(!fs::exists(state)); +} + +TEST_CASE("PIPS_LocalFile_Delta_FallbackLeavesNothingOfTheDeltaAttempt", "[pips][local_file][delta]") +{ + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields(s_Package2Id)); + source.Publish(s_SecondVersion, true); + + // The delta is acquired before anything can tell that its baseline will not serve, so by the + // time the source gives up on the delta it is already being held. + source.RepublishBaselineAtVersion(s_UnexpectedBaselineVersion); + + source.CleanSources(); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + // Acquiring the full index must not leave the unused delta attempt in any state directory. + REQUIRE(!fs::exists(GetStatePathFor(source.DeltaFamilyName))); + + REQUIRE(fs::exists(source.StatePath() / s_IndexMsixName)); + REQUIRE(!fs::exists(source.StatePath() / s_DeltaMsixName)); + REQUIRE(!fs::exists(source.StatePath() / s_BaselineMsixName)); +} + +TEST_CASE("PIPS_LocalFile_Delta_UpdateDiscardsTheFullIndex", "[pips][local_file][delta]") +{ + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields(s_Package2Id)); + source.Publish(s_SecondVersion, true); + + source.CleanSources(); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + // Stands in for a full index acquired while delta was disabled. The two are separate packages, + // so nothing about acquiring a delta would otherwise displace the full index. + fs::path fullIndex = source.StatePath() / s_IndexMsixName; + fs::copy_file(source.Remote.GetPath() / std::string{ s_RemoteIndexMsixName }, fullIndex); + REQUIRE(fs::exists(fullIndex)); + + source.AddPackage(MakeIndexFields(s_Package3Id)); + source.Publish(s_ThirdVersion, true); + + REQUIRE(UpdateSource(details.Name, callback)); + + REQUIRE(!fs::exists(fullIndex)); + + // The packages the source is actually made of are untouched by the housekeeping. + REQUIRE(fs::exists(source.StatePath() / s_DeltaMsixName)); + REQUIRE(fs::exists(source.StatePath() / s_BaselineMsixName)); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); +} + +// The two cases below cover the service rolling its baseline onto a new lineage, which it is free +// to do because the delta names the location and version of the baseline it was computed against. +// +// They are the two halves of a proposed scheme in which the service keeps two overlapping baseline +// lineages so that a baseline can be delivered to a client out of band, ahead of the client ever +// asking for it. Only what exists today is modelled: a baseline carries no link to its own delta +// and no expiration, so what is covered is that a client which already holds the baseline a delta +// names does not acquire it again, and that one which does not acquires it through the ordinary +// flow. + +namespace +{ + // A delta capable source holding two packages, published and added to the client, ready for + // the service to roll its baseline. + struct BaselineRollTest + { + TestHook::SetSingleExperimentalFeature_Override DeltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override TrustOverride; + + TestPreIndexedSource Source{ { MakeIndexFields(s_Package1Id) } }; + SourceDetails Details; + TestProgress Callback; + + BaselineRollTest() + { + Source.PublishBaseline(); + + Source.AddPackage(MakeIndexFields(s_Package2Id)); + Source.Publish(s_SecondVersion, true); + + Source.CleanSources(); + + Details = Source.MakeDetails(); + REQUIRE(AddSource(Details, Callback)); + + REQUIRE(fs::exists(BaselinePackage())); + REQUIRE(GetSourcePackageIds(Details.Name) == std::set{ s_Package1Id, s_Package2Id }); + } + + // Publishes the next baseline lineage beside the current one. The previous lineage stays + // published, as it would while the two overlap. + void PublishNextBaseline() + { + Source.AddPackage(MakeIndexFields(s_Package3Id)); + Source.PublishBaseline(Source.NextBaseline); + } + + // Recomputes the delta at the fixed name against the next baseline, which is what moves + // the client onto that lineage. + void PublishDeltaAgainstNextBaseline() + { + Source.AddPackage(MakeIndexFields(s_Package4Id)); + Source.Publish(Source.NextBaseline, s_ThirdVersion, true); + } + + void RollBaseline() + { + PublishNextBaseline(); + PublishDeltaAgainstNextBaseline(); + } + + // Retires the lineage the client is on, so that the baseline its delta names can no longer + // be acquired and the delta it holds can never be made to pair with anything. + void WithdrawCurrentBaseline() + { + fs::remove(Source.RemotePathFor(Source.Baseline)); + } + + // Puts the next baseline into the client's state without going through the source update + // flow at all, which is what an out of band delivery channel would do. + void DeliverBaselineOutOfBand() + { + fs::copy_file(Source.RemotePathFor(Source.NextBaseline), BaselinePackage(), fs::copy_options::overwrite_existing); + } + + // Marks the held baseline so that a later acquisition of it is detectable. + // + // A write time carried over from the file as it was found is not enough on its own: the + // store persists a local source by copying, and a copy preserves the time, so re-acquiring + // the very same published package would reproduce it exactly. Stamping a time that no + // published package has removes that coincidence. + fs::file_time_type StampBaselineWriteTime() const + { + fs::file_time_type stamp = fs::file_time_type::clock::now() - std::chrono::hours{ 48 }; + fs::last_write_time(BaselinePackage(), stamp); + return fs::last_write_time(BaselinePackage()); + } + + fs::path BaselinePackage() const + { + return Source.StatePath() / s_BaselineMsixName; + } + }; +} + +TEST_CASE("PIPS_LocalFile_Delta_OutOfBandBaselineIsNotReacquired", "[pips][local_file][delta]") +{ + BaselineRollTest test; + + test.RollBaseline(); + test.DeliverBaselineOutOfBand(); + + auto baselineWriteTime = test.StampBaselineWriteTime(); + + // The delta that this acquires names the baseline the client already holds, so the baseline is + // never probed and never downloaded. + REQUIRE(UpdateSource(test.Details.Name, test.Callback)); + REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); + + // The third package is only in the new baseline and the fourth only in the delta, so finding + // both proves that the out of band baseline was paired with the new delta rather than that + // either was read on its own. An open against the previous baseline would have been refused + // outright, since the delta names the identifier of the one it was computed against. + REQUIRE(GetSourcePackageIds(test.Details.Name) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id }); + + // The same again for an update that the open itself decides is due, rather than an explicit + // one. + test.Source.AddPackage(MakeIndexFields(s_Package5Id)); + test.Source.Publish(test.Source.NextBaseline, s_FourthVersion, true); + + REQUIRE(GetSourcePackageIds(test.Details.Name, TimeSpan{ 1 }) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id, s_Package5Id }); + + REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); + + // And an open with nothing new to acquire still leaves it alone. + REQUIRE(GetSourcePackageIds(test.Details.Name) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id, s_Package5Id }); + + REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); +} + +TEST_CASE("PIPS_LocalFile_Delta_RolledBaselineUpdate", "[pips][local_file][delta]") +{ + BaselineRollTest test; + + test.RollBaseline(); + + auto baselineWriteTime = test.StampBaselineWriteTime(); + + // Nothing delivered the new baseline, so the update has to acquire it through the ordinary + // flow. This is the window that an out of band delivery exists to close, and it is what makes + // the case above non vacuous: without the delivery the baseline really does move. + REQUIRE(UpdateSource(test.Details.Name, test.Callback)); + REQUIRE(fs::last_write_time(test.BaselinePackage()) != baselineWriteTime); + + REQUIRE(GetSourcePackageIds(test.Details.Name) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id }); +} + +TEST_CASE("PIPS_LocalFile_Delta_OpenRepairsBaselineAheadOfDelta", "[pips][local_file][delta]") +{ + BaselineRollTest test; + + test.RollBaseline(); + test.DeliverBaselineOutOfBand(); + + auto baselineWriteTime = test.StampBaselineWriteTime(); + + // The delivery put the next lineage's baseline in place while the delta held still names the + // previous one, so what the client holds cannot be opened as a pair. Nothing asks for an + // update here; the open itself has to bring the delta forward. The third package is only in + // the new baseline and the fourth only in the new delta, so finding both proves that the + // delivered baseline was paired with the delta that the open acquired. + REQUIRE(GetSourcePackageIds(test.Details.Name, TimeSpan{ 1 }) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id }); + + // Only the delta was missing. The baseline that the delivery left behind is the one the delta + // names, so it is neither probed nor downloaded again. + REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); +} + +// The cases below cover a source moving between the two stores it can be held in. Which store a +// process writes to is decided by whether it is running for an interactively logged on user, so +// the same source on the same machine can legitimately be maintained in both: an interactive +// winget deploys, while a scheduled task cannot and falls back to our own local state. + +namespace +{ + // A source added by a process that can deploy, holding the overrides that make both stores + // reachable for as long as the test needs them. + struct StoreSwapTest + { + // Held off explicitly, so that these cover the stores rather than whatever the machine + // running the test happens to have enabled. + TestHook::SetSingleExperimentalFeature_Override DeltaDisabled{ ExperimentalFeature::Feature::DeltaIndex, false }; + TestHook::SetSourcePackageTrustValidation_Override TrustOverride; + TestHook::SetDeployedPackageStore_Override DeployedStore{ CreateTestDeployedPackageStore }; + TestHook::SetIsRunningAsInteractiveUser_Override Interactive{ true }; + + TestPreIndexedSource Source{ { MakeIndexFields(s_Package1Id) } }; + SourceDetails Details; + TestProgress Callback; + + StoreSwapTest() + { + Source.Publish(s_FirstVersion); + + Source.CleanSources(); + + Details = Source.MakeDetails(); + REQUIRE(AddSource(Details, Callback)); + } + + // Publishes a full index at a new version, holding one more package than the last one did. + void PublishNewVersion(std::string_view version, const std::string& addedPackageId) + { + Source.AddPackage(MakeIndexFields(addedPackageId)); + Source.Publish(version); + } + + bool Update() + { + return UpdateSource(Details.Name, Callback); + } + + std::set PackageIds() + { + return GetSourcePackageIds(Details.Name); + } + }; +} + +TEST_CASE("PIPS_StoreSwap_WriteFollowsInteractiveUser", "[pips][local_file][store_swap]") +{ + StoreSwapTest test; + + // An interactively logged on user can deploy, so the source is held in the deployed store and + // our own local state is not used at all. + fs::path deployedIndex = test.Source.DeployedStatePath() / s_IndexMsixName; + REQUIRE(fs::exists(deployedIndex)); + REQUIRE(!fs::exists(test.Source.StatePath())); + + std::string deployedContents = GetContents(deployedIndex); + + // A process that cannot deploy maintains the same source in the local file store instead, + // without disturbing what the other store holds. + test.PublishNewVersion(s_SecondVersion, s_Package2Id); + test.Interactive.Set(false); + + REQUIRE(test.Update()); + + fs::path localIndex = test.Source.StatePath() / s_IndexMsixName; + REQUIRE(fs::exists(localIndex)); + REQUIRE(GetContents(deployedIndex) == deployedContents); + REQUIRE(GetContents(localIndex) != deployedContents); +} + +TEST_CASE("PIPS_StoreSwap_ReadFindsFreshestStore", "[pips][local_file][store_swap]") +{ + StoreSwapTest test; + + // The local file store moves ahead of the deployed one. + test.PublishNewVersion(s_SecondVersion, s_Package2Id); + test.Interactive.Set(false); + REQUIRE(test.Update()); + + REQUIRE(test.PackageIds() == std::set{ s_Package1Id, s_Package2Id }); + + // A process that can deploy would write to the deployed store, which holds the older copy. + // The read follows the newer one rather than the one that this process maintains. + test.Interactive.Set(true); + + REQUIRE(test.PackageIds() == std::set{ s_Package1Id, s_Package2Id }); + + // The same in the other direction: the deployed store moves ahead, and a process that cannot + // deploy reads it rather than the copy that it maintains itself. + test.PublishNewVersion(s_ThirdVersion, s_Package3Id); + REQUIRE(test.Update()); + + test.Interactive.Set(false); + + REQUIRE(test.PackageIds() == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); +} + +TEST_CASE("PIPS_StoreSwap_ReadKeepsOwnStoreWhenEqual", "[pips][local_file][store_swap]") +{ + StoreSwapTest test; + + // Both stores are brought to the same version, so neither one is newer than the other. + test.Interactive.Set(false); + REQUIRE(test.Update()); + + REQUIRE(fs::exists(test.Source.StatePath() / s_IndexMsixName)); + REQUIRE(fs::exists(test.Source.DeployedStatePath() / s_IndexMsixName)); + + SourceDetails stored = GetStoredDetails(test.Details.Name); + + PackageKey key; + key.Slot = PackageSlot::FullIndex; + key.Identity = test.Source.FullIndexFamilyName; + + Msix::PackageVersion expectedVersion{ std::string{ s_FirstVersion } }; + + // Which store answered is not something a source exposes, so these ask the store directly. + // Whether a read can be served without the lock is the difference between the two mechanisms, + // and the composite has to report the answer of the store it resolved to rather than its own. + { + // A process that cannot deploy stays on the local file store that it maintains, and that + // store has to extract from the held file, so it cannot serve a read unlocked. + auto store = CreateCompositeStore(stored); + + REQUIRE(store->GetVersion(key) == expectedVersion); + REQUIRE(!store->AllowsUnlockedRead()); + } + + test.Interactive.Set(true); + + { + // A process that can deploy stays on the deployed store for the same reason, and that one + // is read in place. + auto store = CreateCompositeStore(stored); + + REQUIRE(store->GetVersion(key) == expectedVersion); + REQUIRE(store->AllowsUnlockedRead()); + } +} + +TEST_CASE("PIPS_StoreSwap_RemoveClearsBothStores", "[pips][local_file][store_swap]") +{ + StoreSwapTest test; + + test.PublishNewVersion(s_SecondVersion, s_Package2Id); + test.Interactive.Set(false); + REQUIRE(test.Update()); + + REQUIRE(fs::exists(test.Source.DeployedStatePath())); + REQUIRE(fs::exists(test.Source.StatePath())); + + REQUIRE(RemoveSource(test.Details.Name, test.Callback)); + + // Clearing only the store that this process writes to would leave a copy behind that a later + // read would still find and answer from. + REQUIRE(!fs::exists(test.Source.StatePath())); + REQUIRE(!fs::exists(test.Source.DeployedStatePath())); +} diff --git a/src/AppInstallerCLITests/SQLiteIndexDelta.cpp b/src/AppInstallerCLITests/SQLiteIndexDelta.cpp index 6cc6cc6412..f600a6f847 100644 --- a/src/AppInstallerCLITests/SQLiteIndexDelta.cpp +++ b/src/AppInstallerCLITests/SQLiteIndexDelta.cpp @@ -1801,6 +1801,73 @@ TEST_CASE("SQLiteIndex_Delta_OpenWithBaseline_RemovedPackageExcluded", "[sqlitei REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id }); } +TEST_CASE("SQLiteIndex_Delta_OpenWithBaseline_TemporaryFilesCleanedUp", "[sqliteindex][V2_1][delta]") +{ + auto disposition = GENERATE(SQLiteStorageBase::OpenDisposition::Read, SQLiteStorageBase::OpenDisposition::Immutable); + bool validBaseline = GENERATE(true, false); + auto p1 = MakePackage("Publisher1.Id", "Package 1"); + auto p2 = MakePackage("Publisher2.Id", "Package 2"); + DeltaTestContext context{ { p1 } }; + context.Add(p2); + context.GenerateDelta(); + + auto extract = [](const std::filesystem::path& source, const std::filesystem::path& target) + { + std::ifstream stream{ source, std::ios::binary }; + REQUIRE(stream.good()); + std::ostringstream contents; + contents << stream.rdbuf(); + REQUIRE_FALSE(stream.bad()); + auto data = contents.str(); + auto file = ManagedFile::CreateWriteLockedFile(target, GENERIC_WRITE, true); + DWORD written = 0; + REQUIRE(WriteFile(file.GetFileHandle(), data.data(), wil::safe_cast(data.size()), &written, nullptr)); + REQUIRE(written == data.size()); + return file; + }; + + TempFile deltaFile{ "delta_extracted"s, ".db"s }; + TempFile baselineFile{ "baseline_extracted"s, ".db"s }; + auto delta = extract(context.DeltaFile.GetPath(), deltaFile.GetPath()); + // The full working index has no baseline designation, so pairing it fails after attachment. + auto baseline = extract(validBaseline ? context.BaselineFile.GetPath() : context.WorkingFile.GetPath(), baselineFile.GetPath()); + + if (validBaseline) + { + SQLiteIndex combined = SQLiteIndex::OpenWithBaseline( + deltaFile.GetPath().u8string(), baselineFile.GetPath().u8string(), disposition, + std::move(delta), std::move(baseline)); + REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id, p2.Id }); + REQUIRE(std::filesystem::exists(deltaFile.GetPath())); + REQUIRE(std::filesystem::exists(baselineFile.GetPath())); + + SECTION("Move construction") + { + SQLiteIndex moved{ std::move(combined) }; + REQUIRE(GetSearchedIds(moved) == std::set{ p1.Id, p2.Id }); + } + + SECTION("Move assignment") + { + combined = context.OpenCombined(disposition); + REQUIRE(GetSearchedIds(combined) == std::set{ p1.Id, p2.Id }); + REQUIRE_FALSE(std::filesystem::exists(deltaFile.GetPath())); + REQUIRE_FALSE(std::filesystem::exists(baselineFile.GetPath())); + } + } + else + { + REQUIRE_THROWS_HR( + SQLiteIndex::OpenWithBaseline( + deltaFile.GetPath().u8string(), baselineFile.GetPath().u8string(), disposition, + std::move(delta), std::move(baseline)), + APPINSTALLER_CLI_ERROR_INDEX_INTEGRITY_COMPROMISED); + } + + REQUIRE_FALSE(std::filesystem::exists(deltaFile.GetPath())); + REQUIRE_FALSE(std::filesystem::exists(baselineFile.GetPath())); +} + TEST_CASE("SQLiteIndex_Delta_UnmarkedBaselineRejected", "[sqliteindex][V2_1][delta]") { DeltaTestContext context; diff --git a/src/AppInstallerCLITests/TestHooks.h b/src/AppInstallerCLITests/TestHooks.h index 396eafc9e1..87415beb0a 100644 --- a/src/AppInstallerCLITests/TestHooks.h +++ b/src/AppInstallerCLITests/TestHooks.h @@ -4,6 +4,7 @@ #include "TestSettings.h" #include +#include #include #include #include @@ -35,6 +36,7 @@ namespace AppInstaller void TestHook_SetPathOverride(PathName target, const Filesystem::PathDetails& details); void TestHook_ClearPathOverrides(); void TestHook_SetIsRunningWithNonDefaultFullToken_Override(bool* value); + void TestHook_SetIsRunningAsInteractiveUser_Override(bool* value); } namespace Repository @@ -55,6 +57,12 @@ namespace AppInstaller void TestHook_SetGetFontRegistryRootFunc(GetFontRegistryRootFunc value); } + namespace Repository::Microsoft::PreIndexed + { + void TestHook_SetSourcePackageTrustValidation_Override(bool* result); + void TestHook_SetDeployedPackageStore_Override(std::function(const SourceDetails&)>* value); + } + namespace Logging { void TestHook_SetTelemetryOverride(std::shared_ptr ttl); @@ -158,6 +166,64 @@ namespace TestHook bool m_status; }; + // Makes every source package validate as trusted, so that the pre-indexed stores can be driven + // against packages built by the test rather than against signed test data. + struct SetSourcePackageTrustValidation_Override + { + SetSourcePackageTrustValidation_Override(bool status = true) : m_status(status) + { + AppInstaller::Repository::Microsoft::PreIndexed::TestHook_SetSourcePackageTrustValidation_Override(&m_status); + } + + ~SetSourcePackageTrustValidation_Override() + { + AppInstaller::Repository::Microsoft::PreIndexed::TestHook_SetSourcePackageTrustValidation_Override(nullptr); + } + + private: + bool m_status; + }; + + // Takes the place of the deployed package store. + struct SetDeployedPackageStore_Override + { + using Factory = std::function(const AppInstaller::Repository::SourceDetails&)>; + + SetDeployedPackageStore_Override(Factory factory) : m_factory(std::move(factory)) + { + AppInstaller::Repository::Microsoft::PreIndexed::TestHook_SetDeployedPackageStore_Override(&m_factory); + } + + ~SetDeployedPackageStore_Override() + { + AppInstaller::Repository::Microsoft::PreIndexed::TestHook_SetDeployedPackageStore_Override(nullptr); + } + + private: + Factory m_factory; + }; + + // Whether the process appears to be running for an interactively logged on user. + struct SetIsRunningAsInteractiveUser_Override + { + SetIsRunningAsInteractiveUser_Override(bool value) : m_value(value) + { + AppInstaller::Runtime::TestHook_SetIsRunningAsInteractiveUser_Override(&m_value); + } + + ~SetIsRunningAsInteractiveUser_Override() + { + AppInstaller::Runtime::TestHook_SetIsRunningAsInteractiveUser_Override(nullptr); + } + + // The point of most of these tests is what happens when this changes, so it is settable + // rather than requiring a new scope. + void Set(bool value) { m_value = value; } + + private: + bool m_value; + }; + struct SetPinningIndex_Override { SetPinningIndex_Override(const std::filesystem::path& indexPath) @@ -453,9 +519,11 @@ namespace TestHook struct SetSingleExperimentalFeature_Override { - SetSingleExperimentalFeature_Override(AppInstaller::Settings::ExperimentalFeature::Feature feature) + // Forcing a feature off matters as much as forcing it on, so that a test of the behavior + // behind a disabled feature does not depend on the settings of the machine running it. + SetSingleExperimentalFeature_Override(AppInstaller::Settings::ExperimentalFeature::Feature feature, bool enabled = true) { - m_overrides[feature] = true; + m_overrides[feature] = enabled; AppInstaller::Settings::SetExperimentalFeatureOverride(&m_overrides); } diff --git a/src/AppInstallerCLITests/TestMsixPackage.cpp b/src/AppInstallerCLITests/TestMsixPackage.cpp new file mode 100644 index 0000000000..4b60305c7e --- /dev/null +++ b/src/AppInstallerCLITests/TestMsixPackage.cpp @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "TestMsixPackage.h" + +#include + +#pragma comment(lib, "urlmon.lib") + +using namespace Microsoft::WRL; +using namespace AppInstaller; + +namespace TestCommon +{ + namespace + { + // A 1x1 PNG. The manifest schema requires a logo and the packaging API requires every file + // the manifest names to be present, so the smallest real image is the cheapest way to + // satisfy both. + constexpr uint8_t s_MinimalPng[] = { + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1F, 0x15, 0xC4, + 0x89, 0x00, 0x00, 0x00, 0x01, 0x73, 0x52, 0x47, 0x42, 0x00, 0xAE, 0xCE, 0x1C, 0xE9, 0x00, 0x00, + 0x00, 0x04, 0x67, 0x41, 0x4D, 0x41, 0x00, 0x00, 0xB1, 0x8F, 0x0B, 0xFC, 0x61, 0x05, 0x00, 0x00, + 0x00, 0x09, 0x70, 0x48, 0x59, 0x73, 0x00, 0x00, 0x0E, 0xC3, 0x00, 0x00, 0x0E, 0xC3, 0x01, 0xC7, + 0x6F, 0xA8, 0x64, 0x00, 0x00, 0x00, 0x0B, 0x49, 0x44, 0x41, 0x54, 0x18, 0x57, 0x63, 0x60, 0x00, + 0x02, 0x00, 0x00, 0x05, 0x00, 0x01, 0xAA, 0xD5, 0xC8, 0x51, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, + 0x4E, 0x44, 0xAE, 0x42, 0x60, 0x82, + }; + + constexpr std::wstring_view s_LogoPath = L"Assets\\AppPackageStoreLogo.png"; + constexpr std::wstring_view s_IndexPath = L"Public\\index.db"; + + // The hash method the block map is built with. The packaging API will not choose one on + // our behalf, so it has to be named explicitly. + constexpr std::wstring_view s_Sha256Uri = L"http://www.w3.org/2001/04/xmlenc#sha256"; + + ComPtr CreateMemoryStream(const void* data, size_t size) + { + ComPtr result; + + THROW_IF_FAILED(CreateStreamOnHGlobal(nullptr, TRUE, &result)); + + if (size) + { + ULONG written = 0; + THROW_IF_FAILED(result->Write(data, static_cast(size), &written)); + THROW_HR_IF(E_UNEXPECTED, written != size); + } + + LARGE_INTEGER start{}; + THROW_IF_FAILED(result->Seek(start, STREAM_SEEK_SET, nullptr)); + + return result; + } + + std::vector ReadFileBytes(const std::filesystem::path& path) + { + std::ifstream stream{ path, std::ios::in | std::ios::binary }; + THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND), !stream); + return std::vector{ std::istreambuf_iterator{ stream }, std::istreambuf_iterator{} }; + } + + void WriteStreamToFile(IStream* stream, const std::filesystem::path& path) + { + LARGE_INTEGER start{}; + THROW_IF_FAILED(stream->Seek(start, STREAM_SEEK_SET, nullptr)); + + STATSTG stat{}; + THROW_IF_FAILED(stream->Stat(&stat, STATFLAG_NONAME)); + + std::vector buffer(static_cast(stat.cbSize.QuadPart)); + + if (!buffer.empty()) + { + ULONG read = 0; + THROW_IF_FAILED(stream->Read(buffer.data(), static_cast(buffer.size()), &read)); + THROW_HR_IF(E_UNEXPECTED, read != buffer.size()); + } + + std::filesystem::create_directories(path.parent_path()); + + std::ofstream file{ path, std::ios::out | std::ios::binary | std::ios::trunc }; + THROW_HR_IF(E_INVALIDARG, !file); + + if (!buffer.empty()) + { + file.write(reinterpret_cast(buffer.data()), buffer.size()); + } + + file.close(); + } + + std::string CreateManifest(const TestIndexPackageDefinition& definition) + { + // Modelled on the checked in test package's manifest, so that a built package and the + // existing test data differ only in the values a test chooses. + std::ostringstream manifest; + manifest << + R"()" "\n" + R"()" "\n" + R"( )" "\n" + R"( )" "\n" + R"( Fake index for tests)" "\n" + R"( Microsoft Corporation)" "\n" + R"( Assets\AppPackageStoreLogo.png)" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"()" "\n"; + + return manifest.str(); + } + } + + std::string CreateTestIndexPackage( + const std::filesystem::path& outputPath, + const std::filesystem::path& indexFile, + const TestIndexPackageDefinition& definition) + { + ComPtr factory; + THROW_IF_FAILED(CoCreateInstance( + __uuidof(AppxFactory), + nullptr, + CLSCTX_INPROC_SERVER, + __uuidof(IAppxFactory), + reinterpret_cast(factory.GetAddressOf()))); + + ComPtr hashMethod; + THROW_IF_FAILED(CreateUri(s_Sha256Uri.data(), Uri_CREATE_CANONICALIZE, 0, &hashMethod)); + + APPX_PACKAGE_SETTINGS settings{}; + settings.forceZip32 = TRUE; + settings.hashMethod = hashMethod.Get(); + + // The package is assembled in memory and written out only once it is complete, so a failure + // partway through leaves nothing at the output path for a later test to pick up. + ComPtr packageStream = CreateMemoryStream(nullptr, 0); + + ComPtr writer; + THROW_IF_FAILED(factory->CreatePackageWriter(packageStream.Get(), &settings, &writer)); + + std::vector indexBytes = ReadFileBytes(indexFile); + ComPtr indexStream = CreateMemoryStream(indexBytes.data(), indexBytes.size()); + + THROW_IF_FAILED(writer->AddPayloadFile( + s_IndexPath.data(), + L"application/octet-stream", + APPX_COMPRESSION_OPTION_NORMAL, + indexStream.Get())); + + ComPtr logoStream = CreateMemoryStream(s_MinimalPng, sizeof(s_MinimalPng)); + + THROW_IF_FAILED(writer->AddPayloadFile( + s_LogoPath.data(), + L"image/png", + APPX_COMPRESSION_OPTION_NONE, + logoStream.Get())); + + std::string manifest = CreateManifest(definition); + ComPtr manifestStream = CreateMemoryStream(manifest.data(), manifest.size()); + + // Close validates the manifest and writes the block map, so anything malformed is reported + // here rather than when the package is later read. + THROW_IF_FAILED(writer->Close(manifestStream.Get())); + + WriteStreamToFile(packageStream.Get(), outputPath); + + Msix::MsixInfo info{ outputPath }; + return Msix::GetPackageFamilyNameFromFullName(info.GetPackageFullName()); + } +} diff --git a/src/AppInstallerCLITests/TestMsixPackage.h b/src/AppInstallerCLITests/TestMsixPackage.h new file mode 100644 index 0000000000..9591623652 --- /dev/null +++ b/src/AppInstallerCLITests/TestMsixPackage.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include +#include + +namespace TestCommon +{ + // Describes a source index package to build. + struct TestIndexPackageDefinition + { + // The identity name. Together with the publisher this determines the package family name. + std::string Name = "AppInstallerCLITestsFakeIndex"; + + // Must be a well formed distinguished name; the publisher hash is computed from it. + std::string Publisher = "CN=Code Sign Test (DO NOT TRUST), O=Microsoft Corporation, L=Redmond, S=Washington, C=US"; + + // Four part version. This is what the source update path compares to decide whether it + // already holds a package, so it is the field most tests will vary. + std::string Version = "1.0.0.0"; + }; + + // Builds an unsigned MSIX at outputPath carrying indexFile as `Public\index.db`, which is where + // the pre-indexed source expects to find an index. + // + // Returns the package family name of the result, read back out of the package that was written + // rather than computed. + std::string CreateTestIndexPackage( + const std::filesystem::path& outputPath, + const std::filesystem::path& indexFile, + const TestIndexPackageDefinition& definition = {}); +} diff --git a/src/AppInstallerCommonCore/Deployment.cpp b/src/AppInstallerCommonCore/Deployment.cpp index 4b054ff742..ae4d850701 100644 --- a/src/AppInstallerCommonCore/Deployment.cpp +++ b/src/AppInstallerCommonCore/Deployment.cpp @@ -74,12 +74,15 @@ namespace AppInstaller::Deployment IAsyncOperationWithProgress StartAddPackage(PackageManager& packageManager, const winrt::Windows::Foundation::Uri& uri, const Options& options) { + DeploymentOptions deploymentOptions = options.AllowDowngrade ? DeploymentOptions::ForceUpdateFromAnyVersion : DeploymentOptions::None; + if (!options.ExpectedDigests.empty()) { // Must use API that supports digests THROW_WIN32_IF(ERROR_NOT_SUPPORTED, !IsExpectedDigestsSupported()); AddPackageOptions addPackageOptions; + addPackageOptions.ForceUpdateFromAnyVersion(options.AllowDowngrade); for (const auto& digest : options.ExpectedDigests) { @@ -93,7 +96,7 @@ namespace AppInstaller::Deployment return packageManager.AddPackageAsync( uri, nullptr, /*dependencyPackageUris*/ - DeploymentOptions::None, + deploymentOptions, nullptr, /*targetVolume*/ nullptr, /*optionalAndRelatedPackageFamilyNames*/ nullptr, /*optionalPackageUris*/ @@ -104,7 +107,7 @@ namespace AppInstaller::Deployment return packageManager.RequestAddPackageAsync( uri, nullptr, /*dependencyPackageUris*/ - DeploymentOptions::None, + deploymentOptions, nullptr, /*targetVolume*/ nullptr, /*optionalAndRelatedPackageFamilyNames*/ nullptr /*relatedPackageUris*/); @@ -138,7 +141,7 @@ namespace AppInstaller::Deployment std::ostream& operator<<(std::ostream& out, const Options& options) { - out << " { SkipReputationCheck = " << options.SkipReputationCheck << ", ExpectedDigests = {"; + out << " { SkipReputationCheck = " << options.SkipReputationCheck << ", AllowDowngrade = " << options.AllowDowngrade << ", ExpectedDigests = {"; for (const auto& digest : options.ExpectedDigests) { @@ -202,12 +205,12 @@ namespace AppInstaller::Deployment size_t id = GetDeploymentOperationId(); AICLI_LOG(Core, Info, << "Starting RequestAddPackageAsync operation #" << id << ": " << uri); - DeploymentOptions deploymentOptions = DeploymentOptions::None; + DeploymentOptions deploymentOptions = options.AllowDowngrade ? DeploymentOptions::ForceUpdateFromAnyVersion : DeploymentOptions::None; // Optimization to keep files if the package is in use. Only available in a newer OS per: // https://docs.microsoft.com/en-us/uwp/api/Windows.Management.Deployment.DeploymentOptions if (Runtime::IsCurrentOSVersionGreaterThanOrEqual(Utility::Version{ "10.0.18362.0" })) { - deploymentOptions = DeploymentOptions::RetainFilesOnFailure; + deploymentOptions |= DeploymentOptions::RetainFilesOnFailure; } IAsyncOperationWithProgress deployOperation = packageManager.RequestAddPackageAsync( @@ -247,7 +250,10 @@ namespace AppInstaller::Deployment AICLI_LOG(Core, Info, << "Starting RegisterPackageByFullNameAsync operation #" << id << ": " << packageFullName); IAsyncOperationWithProgress registerOperation = - packageManager.RegisterPackageByFullNameAsync(packageFullNameWide, nullptr, DeploymentOptions::None); + packageManager.RegisterPackageByFullNameAsync( + packageFullNameWide, + nullptr, + options.AllowDowngrade ? DeploymentOptions::ForceUpdateFromAnyVersion : DeploymentOptions::None); HRESULT hr = WaitForDeployment(registerOperation, id, progress, false); if (hr == HRESULT_FROM_WIN32(ERROR_PACKAGES_IN_USE)) @@ -342,7 +348,10 @@ namespace AppInstaller::Deployment AICLI_LOG(Core, Info, << "Starting RegisterPackageByFullNameAsync operation #" << id << ": " << packageFullName); IAsyncOperationWithProgress registerOperation = - packageManager.RegisterPackageByFullNameAsync(packageFullNameWide, nullptr, DeploymentOptions::None); + packageManager.RegisterPackageByFullNameAsync( + packageFullNameWide, + nullptr, + options.AllowDowngrade ? DeploymentOptions::ForceUpdateFromAnyVersion : DeploymentOptions::None); WaitForDeployment(registerOperation, id, progress); } catch (...) diff --git a/src/AppInstallerCommonCore/ExperimentalFeature.cpp b/src/AppInstallerCommonCore/ExperimentalFeature.cpp index b59905f0b9..623b04dcfe 100644 --- a/src/AppInstallerCommonCore/ExperimentalFeature.cpp +++ b/src/AppInstallerCommonCore/ExperimentalFeature.cpp @@ -67,6 +67,8 @@ namespace AppInstaller::Settings return userSettings.Get(); case ExperimentalFeature::Feature::InteractivePackageSelection: return userSettings.Get(); + case ExperimentalFeature::Feature::DeltaIndex: + return userSettings.Get(); default: THROW_HR(E_UNEXPECTED); } @@ -102,6 +104,8 @@ namespace AppInstaller::Settings return ExperimentalFeature{ "Font", "fonts", "https://aka.ms/winget-settings", Feature::Font }; case Feature::InteractivePackageSelection: return ExperimentalFeature{ "Interactive Package Selection", "interactivePackageSelection", "https://aka.ms/winget-settings", Feature::InteractivePackageSelection }; + case Feature::DeltaIndex: + return ExperimentalFeature{ "Delta Index", "deltaIndex", "https://aka.ms/winget-settings", Feature::DeltaIndex }; default: THROW_HR(E_UNEXPECTED); } diff --git a/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h b/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h index 40055f4d2f..4dddebe99c 100644 --- a/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h +++ b/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h @@ -16,6 +16,10 @@ namespace AppInstaller::Deployment // Avoid using APIs that make a reputation check. bool SkipReputationCheck = false; + // Allow the deployed version to move backward. + // Deployment refuses a package whose version is lower than the installed one unless this is set. + bool AllowDowngrade = false; + // The pairs of URI+Digest to enforce. std::vector> ExpectedDigests; }; diff --git a/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h b/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h index 436196cdc3..4146123782 100644 --- a/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h +++ b/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h @@ -26,6 +26,7 @@ namespace AppInstaller::Settings Resume = 0x2, Font = 0x4, InteractivePackageSelection = 0x8, + DeltaIndex = 0x10, Max, // This MUST always be after all experimental features // Features listed after Max will not be shown with the features command diff --git a/src/AppInstallerCommonCore/Public/winget/UserSettings.h b/src/AppInstallerCommonCore/Public/winget/UserSettings.h index e000c6adda..64e1d6a459 100644 --- a/src/AppInstallerCommonCore/Public/winget/UserSettings.h +++ b/src/AppInstallerCommonCore/Public/winget/UserSettings.h @@ -103,6 +103,7 @@ namespace AppInstaller::Settings EFResume, EFFonts, EFInteractivePackageSelection, + EFDeltaIndex, // Telemetry TelemetryDisable, // Install behavior @@ -196,6 +197,7 @@ namespace AppInstaller::Settings SETTINGMAPPING_SPECIALIZATION(Setting::EFResume, bool, bool, false, ".experimentalFeatures.resume"sv); SETTINGMAPPING_SPECIALIZATION(Setting::EFFonts, bool, bool, false, ".experimentalFeatures.fonts"sv); SETTINGMAPPING_SPECIALIZATION(Setting::EFInteractivePackageSelection, bool, bool, false, ".experimentalFeatures.interactivePackageSelection"sv); + SETTINGMAPPING_SPECIALIZATION(Setting::EFDeltaIndex, bool, bool, false, ".experimentalFeatures.deltaIndex"sv); // Telemetry SETTINGMAPPING_SPECIALIZATION(Setting::TelemetryDisable, bool, bool, false, ".telemetry.disable"sv); // Install behavior diff --git a/src/AppInstallerCommonCore/UserSettings.cpp b/src/AppInstallerCommonCore/UserSettings.cpp index 5f3074f0e8..7c4399838f 100644 --- a/src/AppInstallerCommonCore/UserSettings.cpp +++ b/src/AppInstallerCommonCore/UserSettings.cpp @@ -288,6 +288,7 @@ namespace AppInstaller::Settings WINGET_VALIDATE_PASS_THROUGH(EFResume) WINGET_VALIDATE_PASS_THROUGH(EFFonts) WINGET_VALIDATE_PASS_THROUGH(EFInteractivePackageSelection) + WINGET_VALIDATE_PASS_THROUGH(EFDeltaIndex) WINGET_VALIDATE_PASS_THROUGH(AnonymizePathForDisplay) WINGET_VALIDATE_PASS_THROUGH(TelemetryDisable) WINGET_VALIDATE_PASS_THROUGH(InteractivityDisable) diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj index 61bdddfbdb..0b3d62df38 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -287,6 +287,7 @@ + @@ -430,11 +431,13 @@ + + diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters index 3bfc9f0dac..5effbf6a4f 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -192,6 +192,9 @@ Microsoft\PreIndexed + + Microsoft\PreIndexed + Microsoft @@ -572,6 +575,9 @@ Source Files + + Microsoft\PreIndexed + Microsoft\PreIndexed @@ -587,6 +593,9 @@ Microsoft\PreIndexed + + Microsoft\PreIndexed + Microsoft diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp new file mode 100644 index 0000000000..bce0017247 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp @@ -0,0 +1,439 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/PreIndexed/IndexForm.h" +#include "Microsoft/PreIndexed/RemotePackage.h" +#include "Microsoft/PreIndexed/SourceData.h" + +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + namespace anon + { + // A source composed of a delta and the baseline that the delta names. + struct DeltaIndexForm : public IIndexForm + { + DeltaIndexForm(const SourceDetails& details) : m_details(details), m_data(details.Data) {} + + std::optional DiscoverIdentities(IProgressCallback& progress) override + { + if (!m_data.HasDeltaIdentity()) + { + if (m_data.HasFullIdentity()) + { + // The source is already configured, and what it is configured with names + // no delta. Probing here would be a migration, which is deliberately not + // performed: the identity a probe found could not be written back anyway, + // since an update is given a const SourceDetails and only the metadata + // fields are persisted afterwards. A third party source that starts + // publishing a delta can be re-added. + return std::nullopt; + } + + // Nothing is known about this source yet, so it is being added. That is the + // one operation that can both probe for a delta and store what it finds. + std::optional deltaIdentity = ProbeDeltaIdentity(progress); + + if (!deltaIdentity) + { + // The source publishes no delta. That is an ordinary answer, and the caller + // falls back to the full index rather than treating it as a failure. + return std::nullopt; + } + + m_data.DeltaIdentity(std::move(deltaIdentity.value())); + } + + if (!m_data.HasFullIdentity()) + { + // Record the full identity before acquisition so disabling delta can use it + // without changing the source's identity or probing during an update. + auto fullForm = CreateFullIndexForm(m_details); + auto fullData = fullForm->DiscoverIdentities(progress); + if (fullData) + { + m_data.FullIdentity(SourceData{ fullData.value() }.FullIdentity()); + } + } + + // The baseline is deliberately not probed for here. Where it is published is + // recorded in the delta, so finding it means acquiring and reading one, and the + // update that immediately follows is going to do that anyway. What it learns is + // reported back through UpdateReport::LearnedIdentities. + return m_data.Serialize(); + } + + std::vector GetPackages() const override + { + std::vector result{ GetDeltaKey() }; + + if (m_data.HasBaselineIdentity()) + { + result.emplace_back(GetBaselineKey()); + } + + return result; + } + + bool HasIdentities() const override + { + return m_data.HasDeltaIdentity() && m_data.HasBaselineIdentity(); + } + + bool IsHeld(const IPackageStore& store) const override + { + return m_data.HasBaselineIdentity() && + store.GetVersion(GetDeltaKey()).has_value() && + store.GetVersion(GetBaselineKey()).has_value(); + } + + bool IsUsable(IPackageStore& store, IProgressCallback& progress) override + { + if (!m_data.HasBaselineIdentity()) + { + // Which baseline this source uses has never been learned, so there is nothing + // held that could be paired with the delta. + return false; + } + + std::optional heldBaselineVersion = store.GetVersion(GetBaselineKey()); + + if (!heldBaselineVersion) + { + return false; + } + + auto locator = ReadBaselineLocator(store, progress); + + if (!locator) + { + // A delta that cannot be read, or that names no baseline, cannot be paired + // with anything. + return false; + } + + // The version is the whole of the question: a package identity and version name + // one set of contents, so a baseline held at the version the delta names is the + // baseline the delta was computed against. + if (heldBaselineVersion.value() != Msix::PackageVersion{ locator->PackageVersion }) + { + AICLI_LOG(Repo, Warning, << "Delta for source `" << m_details.Name << "` names baseline version " << + locator->PackageVersion << ", but the baseline held is version " << heldBaselineVersion.value().ToString()); + return false; + } + + return true; + } + + std::optional GetHeldVersion(const IPackageStore& store) const override + { + return store.GetVersion(GetDeltaKey()); + } + + UpdateResult Update(IPackageStore& store, bool isBackground, IProgressCallback& progress, UpdateReport& report) override + { + report.UsedDeltaDownload = true; + + std::optional currentDeltaVersion = store.GetVersion(GetDeltaKey()); + + if (currentDeltaVersion) + { + report.PreviousIndexPublishedAt = Utility::GetTimePointFromVersion(currentDeltaVersion.value()); + } + + // The delta is published at a fixed name beside the full index, and is probed by + // exactly the same mechanism. + PreIndexedPackageUpdateCheck deltaCheck(GetDeltaPackageLocations(m_details)); + + report.NewIndexPublishedAt = Utility::GetTimePointFromVersion(deltaCheck.AvailableVersion()); + + bool deltaIsCurrent = currentDeltaVersion && currentDeltaVersion.value() >= deltaCheck.AvailableVersion(); + + std::optional locator; + + if (deltaIsCurrent) + { + // The delta we hold is current, but we may still be missing the baseline that + // it names, so being up to date is not on its own a reason to stop. + locator = ReadBaselineLocator(store, progress); + + if (locator) + { + auto heldBaselineVersion = GetHeldBaselineVersion(store); + + if (heldBaselineVersion && heldBaselineVersion.value() == Msix::PackageVersion{ locator->PackageVersion }) + { + AICLI_LOG(Repo, Verbose, << "Remote delta (" << deltaCheck.AvailableVersion().ToString() << + ") was not newer than existing (" << currentDeltaVersion.value().ToString() << + ") and its baseline is held, no update needed"); + + report.PreviousBaselinePublishedAt = Utility::GetTimePointFromVersion(heldBaselineVersion.value()); + report.NewBaselinePublishedAt = report.PreviousBaselinePublishedAt; + return UpdateResult::Success; + } + } + } + + // Re-acquire the delta when it is not current, and also when we could not read the + // one we hold -- in that case what we hold is unusable whatever its version says. + bool acquireDelta = !deltaIsCurrent || !locator; + + if (progress.IsCancelledBy(CancelReason::Any)) + { + AICLI_LOG(Repo, Info, << "Cancelling update upon request"); + return UpdateResult::Aborted; + } + + auto lock = store.Lock(progress, isBackground); + if (!lock) + { + // The delta may well have been usable, so falling back here would acquire a + // full index that the source did not need. + return UpdateResult::Aborted; + } + + std::optional deltaBytes; + + if (acquireDelta) + { + auto acquired = store.Acquire(GetDeltaKey(), deltaCheck.PackageLocation(), progress); + if (!acquired) + { + // Acquisition reports nothing only when it was cancelled. + return UpdateResult::Aborted; + } + + deltaBytes = acquired->DownloadedBytes; + + store.Persist(std::move(acquired.value()), progress); + + locator = ReadBaselineLocator(store, progress); + } + + if (!locator) + { + AICLI_LOG(Repo, Warning, << "Delta for source `" << m_details.Name << "` did not name a baseline"); + return UpdateResult::Unusable; + } + + AICLI_LOG(Repo, Info, << "Delta for source `" << m_details.Name << "` names baseline " << locator->Identifier << + " at `" << locator->RelativeSourcePath << "` version " << locator->PackageVersion); + + std::vector baselineLocations = GetBaselinePackageLocations(m_details, locator->RelativeSourcePath); + + if (!m_data.HasBaselineIdentity()) + { + // The source is being added. Only now, with a delta in hand to say where the + // baseline is published, can the identity to acquire it under be read. + if (!LearnBaselineIdentity(baselineLocations)) + { + return UpdateResult::Unusable; + } + + report.LearnedIdentities = m_data.Serialize(); + } + + Msix::PackageVersion requiredBaselineVersion{ locator->PackageVersion }; + std::optional currentBaselineVersion = store.GetVersion(GetBaselineKey()); + + if (currentBaselineVersion) + { + report.PreviousBaselinePublishedAt = Utility::GetTimePointFromVersion(currentBaselineVersion.value()); + } + + report.NewBaselinePublishedAt = Utility::GetTimePointFromVersion(requiredBaselineVersion); + + std::optional baselineBytes; + + if (currentBaselineVersion && currentBaselineVersion.value() == requiredBaselineVersion) + { + AICLI_LOG(Repo, Verbose, << "Already holding baseline version " << requiredBaselineVersion.ToString()); + } + else + { + PreIndexedPackageUpdateCheck baselineCheck(baselineLocations); + + // The delta told us which version it was computed against; anything else at + // that location is not the baseline this delta can be paired with. + if (baselineCheck.AvailableVersion() != requiredBaselineVersion) + { + AICLI_LOG(Repo, Warning, << "Baseline at `" << baselineCheck.PackageLocation() << "` was version " << + baselineCheck.AvailableVersion().ToString() << ", but the delta named " << requiredBaselineVersion.ToString()); + return UpdateResult::Unusable; + } + + auto acquired = store.Acquire(GetBaselineKey(), baselineCheck.PackageLocation(), progress); + if (!acquired) + { + // Acquisition reports nothing only when it was cancelled. + return UpdateResult::Aborted; + } + + baselineBytes = acquired->DownloadedBytes; + + store.Persist(std::move(acquired.value()), progress); + + report.BaselineUpdated = true; + } + + if (deltaBytes || baselineBytes) + { + report.DownloadedBytes = deltaBytes.value_or(0) + baselineBytes.value_or(0); + report.Reportable = true; + } + + DiscardFullIndex(store, progress); + + return UpdateResult::Success; + } + + SQLiteIndex Open(IPackageStore& store, IProgressCallback& progress) override + { + auto delta = store.GetIndex(GetDeltaKey(), progress); + auto baseline = store.GetIndex(GetBaselineKey(), progress); + + if (!delta || !baseline) + { + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + return SQLiteIndex::OpenWithBaseline( + delta->Path.u8string(), + baseline->Path.u8string(), + SQLiteIndex::OpenDisposition::Immutable, + std::move(delta->TemporaryFile), + std::move(baseline->TemporaryFile)); + } + + private: + PackageKey GetDeltaKey() const + { + THROW_HR_IF(E_NOT_VALID_STATE, !m_data.HasDeltaIdentity()); + return PackageKey{ PackageSlot::Delta, m_data.DeltaIdentity() }; + } + + PackageKey GetBaselineKey() const + { + THROW_HR_IF(E_NOT_VALID_STATE, !m_data.HasBaselineIdentity()); + return PackageKey{ PackageSlot::Baseline, m_data.BaselineIdentity() }; + } + + // The version of the baseline that the store holds, or nothing when there is none -- + // including when which baseline this source uses has never been learned. + std::optional GetHeldBaselineVersion(const IPackageStore& store) const + { + return m_data.HasBaselineIdentity() ? store.GetVersion(GetBaselineKey()) : std::nullopt; + } + + // Learns the identity to acquire the baseline under by reading the published package. + // + // False when it could not be read, which leaves the source with a delta it has no way + // to pair. + bool LearnBaselineIdentity(const std::vector& baselineLocations) + { + try + { + PreIndexedPackageInfo packageInfo(baselineLocations, [](const std::string& packageLocation) + { + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); + }); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); + + m_data.BaselineIdentity(Msix::GetPackageFamilyNameFromFullName(packageInfo.MsixInfo().GetPackageFullName())); + + AICLI_LOG(Repo, Info, << "Baseline for source `" << m_details.Name << "` has identity " << m_data.BaselineIdentity()); + return true; + } + catch (...) + { + LOG_CAUGHT_EXCEPTION_MSG("Could not read the baseline named by the delta for source: %hs", m_details.Name.c_str()); + return false; + } + } + + // Discards the full index, which a source composed of a delta and a baseline has no + // use for. + // + // The baseline is a package in its own right rather than something published over the + // full index, so nothing displaces one that was acquired before this source became a + // delta pair, or by a client that had fallen back to it. Left alone it would sit there + // for as long as the source existed, at the size that having a delta at all is meant + // to avoid. + // + // Best effort: the update has already succeeded, and failing it over housekeeping + // would leave the source no better off. + void DiscardFullIndex(IPackageStore& store, IProgressCallback& progress) + { + try + { + store.RemovePackage(PackageKey{ PackageSlot::FullIndex, m_data.FullIdentity() }, progress); + } + CATCH_LOG(); + } + + // Learns the delta's identity by reading the published package. + // + // Nothing when the source publishes no delta at that location, which is how a source + // that is not delta capable is recognized. + std::optional ProbeDeltaIdentity(IProgressCallback&) + { + try + { + PreIndexedPackageInfo packageInfo(GetDeltaPackageLocations(m_details), [](const std::string& packageLocation) + { + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); + }); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); + + return Msix::GetPackageFamilyNameFromFullName(packageInfo.MsixInfo().GetPackageFullName()); + } + catch (...) + { + LOG_CAUGHT_EXCEPTION_MSG("No delta found for source: %hs", m_details.Name.c_str()); + return std::nullopt; + } + } + + // Reads the baseline that the delta we hold names, from the delta itself. + // + // Nothing when the delta cannot be read or does not carry a locator; the caller falls + // back to the full index rather than guessing at a baseline of its own. + std::optional ReadBaselineLocator(IPackageStore& store, IProgressCallback& progress) + { + try + { + auto extracted = store.GetIndex(GetDeltaKey(), progress); + if (!extracted) + { + return std::nullopt; + } + + SQLiteIndex deltaIndex = SQLiteIndex::Open(extracted->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable); + return deltaIndex.GetDeltaBaselineLocator(); + } + catch (...) + { + if (progress.IsCancelledBy(CancelReason::Any)) + { + throw; + } + + LOG_CAUGHT_EXCEPTION_MSG("Could not read the baseline locator from the held delta"); + return std::nullopt; + } + } + + SourceDetails m_details; + SourceData m_data; + }; + } + + std::unique_ptr CreateDeltaIndexForm(const SourceDetails& details) + { + return std::make_unique(details); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp index 5262f2587f..9b3d6b27b5 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp @@ -19,6 +19,13 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { Deployment::Options options{ IsTrusted() }; + // A baseline is acquired because a delta names it, not because it is the newest one + // published. The delta a client holds can name an older baseline than the one it + // already has -- after a baseline arrives ahead of the delta computed against it, + // for instance -- so the move backward has to be permitted or that pair could + // never be formed. + options.AllowDowngrade = (package.Key.Slot == PackageSlot::Baseline); + // The file lock is deliberately held across deployment, so that the package the // platform reads is the one that was validated. winrt::Windows::Foundation::Uri uri = winrt::Windows::Foundation::Uri(package.Path.c_str()); @@ -72,17 +79,30 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { for (const auto& package : packages) { - auto fullName = Msix::GetPackageFullNameFromFamilyName(package.Identity); - - if (!fullName) - { - AICLI_LOG(Repo, Info, << "No full name found for family name: " << package.Identity); - } - else - { - AICLI_LOG(Repo, Info, << "Removing package: " << *fullName); - Deployment::RemovePackage(*fullName, winrt::Windows::Management::Deployment::RemovalOptions::None, progress); - } + RemovePackage(package, progress); + } + } + + void RemovePackage(const PackageKey& package, IProgressCallback& progress) override + { + if (package.Identity.empty()) + { + // A deployed package can only be found by identity, so there is nothing to act + // on. This is reachable for the full index of a source that never had one. + AICLI_LOG(Repo, Info, << "No identity known for the " << GetSlotName(package.Slot) << " package, nothing to remove"); + return; + } + + auto fullName = Msix::GetPackageFullNameFromFamilyName(package.Identity); + + if (!fullName) + { + AICLI_LOG(Repo, Info, << "No full name found for family name: " << package.Identity); + } + else + { + AICLI_LOG(Repo, Info, << "Removing package: " << *fullName); + Deployment::RemovePackage(*fullName, winrt::Windows::Management::Deployment::RemovalOptions::None, progress); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp index 3eaf4bdbe6..51dce4d99d 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp @@ -3,6 +3,7 @@ #include "pch.h" #include "Microsoft/PreIndexed/IndexForm.h" #include "Microsoft/PreIndexed/RemotePackage.h" +#include "Microsoft/PreIndexed/SourceData.h" #include #include @@ -14,16 +15,13 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // A source composed of a single package holding the complete index. struct FullIndexForm : public IIndexForm { - FullIndexForm(const SourceDetails& details) : m_details(details) - { - m_identity = details.Data; - } + FullIndexForm(const SourceDetails& details) : m_details(details), m_data(details.Data) {} std::optional DiscoverIdentities(IProgressCallback&) override { - if (!m_identity.empty()) + if (m_data.HasFullIdentity()) { - return m_identity; + return m_data.Serialize(); } PreIndexedPackageInfo packageInfo(GetFullIndexPackageLocations(m_details), [](const std::string& packageLocation) @@ -38,9 +36,9 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed auto fullName = packageInfo.MsixInfo().GetPackageFullName(); AICLI_LOG(Repo, Info, << "Found package full name: " << m_details.Name << " => " << fullName); - m_identity = Msix::GetPackageFamilyNameFromFullName(fullName); + m_data.FullIdentity(Msix::GetPackageFamilyNameFromFullName(fullName)); - return m_identity; + return m_data.Serialize(); } std::vector GetPackages() const override @@ -50,7 +48,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed bool HasIdentities() const override { - return !m_identity.empty(); + return m_data.HasFullIdentity(); } bool IsHeld(const IPackageStore& store) const override @@ -58,6 +56,12 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return GetHeldVersion(store).has_value(); } + bool IsUsable(IPackageStore& store, IProgressCallback&) override + { + // There is only one package, so there is nothing for it to disagree with. + return IsHeld(store); + } + std::optional GetHeldVersion(const IPackageStore& store) const override { return store.GetVersion(GetKey()); @@ -146,12 +150,12 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed private: PackageKey GetKey() const { - THROW_HR_IF(E_NOT_VALID_STATE, m_identity.empty()); - return PackageKey{ PackageSlot::FullIndex, m_identity }; + THROW_HR_IF(E_NOT_VALID_STATE, !m_data.HasFullIdentity()); + return PackageKey{ PackageSlot::FullIndex, m_data.FullIdentity() }; } SourceDetails m_details; - std::string m_identity; + SourceData m_data; }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h index 8c72efb4a4..edf58c530b 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h @@ -27,8 +27,19 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // Whether this update is one that should be reported. bool Reportable = false; + + // What the update learned about the source's packages, when that is more than the details + // it was given already said. + // + // A delta names the location of its baseline, so the baseline's identity cannot be known + // until a delta has been acquired and read. An update is therefore the first operation + // that can learn it, and the value to store back into SourceDetails::Data is reported + // here rather than discovered a second time. Only the add path can act on it; every other + // caller is given a const SourceDetails and has nowhere to put it. + std::optional LearnedIdentities; }; + // What an attempt to update a form concluded. enum class UpdateResult { @@ -59,14 +70,18 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // learns. A form whose details are already populated reports what they say, so a source // configured before this form existed is not silently migrated onto it. // + // What is discovered here has to be enough to identify the source and to begin acquiring + // its packages, but need not be all of them: see UpdateReport::LearnedIdentities. + // // Returns the value that should be written to SourceDetails::Data, or nothing when the // source does not publish the packages that this form needs. virtual std::optional DiscoverIdentities(IProgressCallback& progress) = 0; - // The packages that make up this form. - // Throws when the identities are not yet known; see DiscoverIdentities. + // The packages that make up this form, as far as their identities are known. + // Throws when nothing is known at all; see DiscoverIdentities. virtual std::vector GetPackages() const = 0; + // Whether the identities of this form's packages are already known, so that GetPackages // can be called and no probe is needed. virtual bool HasIdentities() const = 0; @@ -74,6 +89,13 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // Whether the store holds every package that this form needs. virtual bool IsHeld(const IPackageStore& store) const = 0; + // Whether the packages that the store holds for this form can be opened together. + // + // IsHeld only asks whether each package is present. A form whose packages have to agree + // with one another can hold a set that does not, and that cannot be answered without + // reading them, so it is a separate and more expensive question. + virtual bool IsUsable(IPackageStore& store, IProgressCallback& progress) = 0; + // The published version of the held index, which is what staleness is judged from. virtual std::optional GetHeldVersion(const IPackageStore& store) const = 0; @@ -89,4 +111,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // The complete index, as acquired by a client that is not using a delta. std::unique_ptr CreateFullIndexForm(const SourceDetails& details); + + // A delta paired with the baseline that it names. + std::unique_ptr CreateDeltaIndexForm(const SourceDetails& details); } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp index c37741644e..a2d32ff48a 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp @@ -162,6 +162,18 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed } } + // Each slot is a file of its own, so one can be discarded without disturbing the rest. + // The identity is not needed; the file is found by slot. + void RemovePackage(const PackageKey& package, IProgressCallback&) override + { + std::filesystem::path packagePath = GetPathForSlot(package.Slot); + + if (std::filesystem::remove(packagePath)) + { + AICLI_LOG(Repo, Info, << "Removed " << GetSlotName(package.Slot) << " package at: " << packagePath.u8string()); + } + } + // Reading extracts from the held file, so the lock must be taken first. bool AllowsUnlockedRead() const override { return false; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp index fd2736a63a..3b81f9e792 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "Microsoft/PreIndexed/PackageStore.h" +#include "Microsoft/PreIndexed/SourceData.h" #include "Microsoft/PreIndexedPackageSourceFactory.h" #include @@ -10,6 +11,10 @@ using namespace std::string_literals; +// Set this to true when test hooks are enabled to force the use of +// the local file package store and disable signing enforcement. +#define WINGET_TEST_HOOK_FORCE_UNSIGNED_FILE_PACKAGE_STORE false + namespace AppInstaller::Repository::Microsoft::PreIndexed { std::string_view GetSlotName(PackageSlot slot) @@ -77,7 +82,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // // The fallback exists because Identifier was not always stored; a source written by an old // enough client has only Data, which is the value such a client would have put there. - m_sourceIdentity = details.Identifier.empty() ? details.Data : details.Identifier; + m_sourceIdentity = details.Identifier.empty() ? SourceData{ details.Data }.SourceIdentity() : details.Identifier; THROW_HR_IF(E_UNEXPECTED, m_sourceIdentity.empty()); } @@ -91,9 +96,35 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return WI_IsFlagSet(m_trustLevel, SourceTrustLevel::Trusted); } +#ifndef AICLI_DISABLE_TEST_HOOKS + static bool* s_SourcePackageTrustValidation_TestHook_Override = nullptr; + + void TestHook_SetSourcePackageTrustValidation_Override(bool* result) + { + s_SourcePackageTrustValidation_TestHook_Override = result; + } +#endif + bool PackageStoreBase::ValidateTrust(const Msix::WriteLockedMsixFile& package) const { - return package.ValidateTrustInfo(RequireStoreOrigin()); +#ifndef AICLI_DISABLE_TEST_HOOKS + // Source packages must be signed, which makes every store path untestable without + // installing a certificate. Overriding the answer here lets the stores be driven against + // packages built on the fly, so that the tests can choose the identities and versions that + // the delta pairing logic turns on. + if (s_SourcePackageTrustValidation_TestHook_Override) + { + return *s_SourcePackageTrustValidation_TestHook_Override; + } + else if constexpr (WINGET_TEST_HOOK_FORCE_UNSIGNED_FILE_PACKAGE_STORE) + { + return true; + } + else +#endif + { + return package.ValidateTrustInfo(RequireStoreOrigin()); + } } std::optional PackageStoreBase::Acquire(const PackageKey& package, const std::string& location, IProgressCallback& progress) @@ -167,13 +198,48 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return result; } - bool CanUseDeployedPackage() +#ifndef AICLI_DISABLE_TEST_HOOKS + static std::function(const SourceDetails&)>* s_DeployedPackageStore_TestHook_Override = nullptr; + + void TestHook_SetDeployedPackageStore_Override(std::function(const SourceDetails&)>* value) { - return Runtime::IsRunningInPackagedContext() && Runtime::IsRunningAsInteractiveUser(); + s_DeployedPackageStore_TestHook_Override = value; } +#endif namespace anon { + // Whether this process can reach a deployed package. + bool CanReachDeployedPackage() + { +#ifndef AICLI_DISABLE_TEST_HOOKS + if (s_DeployedPackageStore_TestHook_Override) + { + return true; + } + else if constexpr (WINGET_TEST_HOOK_FORCE_UNSIGNED_FILE_PACKAGE_STORE) + { + return false; + } + else +#endif + { + return Runtime::IsRunningInPackagedContext(); + } + } + + std::unique_ptr CreateDeployedStore(const SourceDetails& details) + { +#ifndef AICLI_DISABLE_TEST_HOOKS + if (s_DeployedPackageStore_TestHook_Override) + { + return (*s_DeployedPackageStore_TestHook_Override)(details); + } +#endif + + return CreateDeployedPackageStore(details); + } + // A store over every store that this process can reach a source's packages through. // // Reading answers from whichever of them holds the most recently published copy, and @@ -188,15 +254,15 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // recent copy keeps a read on the store that it maintains itself. if (CanUseDeployedPackage()) { - m_stores.emplace_back(CreateDeployedPackageStore(details)); + m_stores.emplace_back(CreateDeployedStore(details)); m_stores.emplace_back(CreateLocalFilePackageStore(details)); } else { m_stores.emplace_back(CreateLocalFilePackageStore(details)); - if (Runtime::IsRunningInPackagedContext()) + if (CanReachDeployedPackage()) { - m_stores.emplace_back(CreateDeployedPackageStore(details)); + m_stores.emplace_back(CreateDeployedStore(details)); } } } @@ -249,7 +315,23 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed THROW_IF_FAILED(firstStoreFailure); } + void RemovePackage(const PackageKey& package, IProgressCallback& progress) override + { + // Every store that could hold it drops it. A package the source has stopped + // needing is unwanted wherever it is, and unlike a read there is no single store + // that holds the answer. + for (auto& store : m_stores) + { + try + { + store->RemovePackage(package, progress); + } + CATCH_LOG(); + } + } + Synchronization::CrossProcessLock Lock(IProgressCallback& progress, bool isBackground = false) override + { // The lock is named from the source rather than from the mechanism, so every store // for a source shares one and any of them can take it. @@ -296,9 +378,23 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed }; } + bool CanUseDeployedPackage() + { +#ifndef AICLI_DISABLE_TEST_HOOKS + if constexpr (WINGET_TEST_HOOK_FORCE_UNSIGNED_FILE_PACKAGE_STORE) + { + return false; + } + else +#endif + { + return anon::CanReachDeployedPackage() && Runtime::IsRunningAsInteractiveUser(); + } + } + std::unique_ptr CreateStore(const SourceDetails& details) { - return CanUseDeployedPackage() ? CreateDeployedPackageStore(details) : CreateLocalFilePackageStore(details); + return CanUseDeployedPackage() ? anon::CreateDeployedStore(details) : CreateLocalFilePackageStore(details); } std::unique_ptr CreateCompositeStore(const SourceDetails& details) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h index 1fdf292694..bf0bbe701f 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h @@ -138,6 +138,17 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // Removes everything held for the given packages. virtual void Remove(const std::vector& packages, IProgressCallback& progress) = 0; + // Removes just the one package, leaving everything else the source holds in place. + // + // This is what discards a package that a source has stopped needing, which Remove cannot + // express: Remove is given every package a source could have and is free to clear the + // source wholesale, and a store that holds its packages together does exactly that. + // + // Removing what is not held succeeds. A store that cannot act on the key it is given -- + // one that finds packages by identity, when the identity is not known -- logs and leaves + // it, since the alternative is to fail an operation that has otherwise succeeded. + virtual void RemovePackage(const PackageKey& package, IProgressCallback& progress) = 0; + // Takes the cross process lock guarding this source's local copy. virtual Synchronization::CrossProcessLock Lock(IProgressCallback& progress, bool isBackground = false) = 0; diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.cpp new file mode 100644 index 0000000000..29a4fac0ea --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.cpp @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/PreIndexed/SourceData.h" + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + using namespace std::string_view_literals; + + namespace + { + constexpr std::string_view s_SourceData_FullIdentity = "fullIdentity"sv; + constexpr std::string_view s_SourceData_DeltaIdentity = "deltaIdentity"sv; + constexpr std::string_view s_SourceData_BaselineIdentity = "baselineIdentity"sv; + + // Reads an optional string member, requiring that it be a string when present. + std::string ReadStringMember(const Json::Value& root, std::string_view name) + { + const Json::Value& member = root[std::string{ name }]; + + if (member.isNull()) + { + return {}; + } + + if (!member.isString()) + { + AICLI_LOG(Repo, Error, << "Source data member '" << name << "' was not a string"); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + return member.asString(); + } + } + + SourceData::SourceData(std::string_view data) + { + size_t firstNonSpace = data.find_first_not_of(" \t\r\n"sv); + + if (firstNonSpace == std::string_view::npos) + { + // An empty value names nothing; the identities are discovered when the source is added. + return; + } + + if (data[firstNonSpace] != '{') + { + // The original syntax: the whole value is the one identity that the source has. + m_fullIdentity = data; + return; + } + + Json::Value root; + Json::CharReaderBuilder builder; + const std::unique_ptr reader{ builder.newCharReader() }; + std::string error; + + if (!reader->parse(data.data(), data.data() + data.size(), &root, &error)) + { + AICLI_LOG(Repo, Error, << "Source data was not valid JSON (" << error << "): " << data); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + if (!root.isObject()) + { + AICLI_LOG(Repo, Error, << "Source data was not a JSON object: " << data); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + // Unrecognized members are ignored, so that a newer client can add to this value without + // an older one refusing to read the source at all. + m_fullIdentity = ReadStringMember(root, s_SourceData_FullIdentity); + m_deltaIdentity = ReadStringMember(root, s_SourceData_DeltaIdentity); + m_baselineIdentity = ReadStringMember(root, s_SourceData_BaselineIdentity); + + // A value that names no package at all identifies nothing, and every path from here needs + // something to key the source on. + if (m_fullIdentity.empty() && m_deltaIdentity.empty()) + { + AICLI_LOG(Repo, Error, << "Source data did not contain an identity for the source: " << data); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + } + + std::string SourceData::Serialize() const + { + if (m_deltaIdentity.empty() && m_baselineIdentity.empty()) + { + return m_fullIdentity; + } + + Json::Value root{ Json::objectValue }; + + // Only what the source actually has is written, so that the value says what it means + // rather than carrying empty members that a reader would have to interpret. + if (!m_fullIdentity.empty()) + { + root[std::string{ s_SourceData_FullIdentity }] = m_fullIdentity; + } + + if (!m_deltaIdentity.empty()) + { + root[std::string{ s_SourceData_DeltaIdentity }] = m_deltaIdentity; + } + + if (!m_baselineIdentity.empty()) + { + root[std::string{ s_SourceData_BaselineIdentity }] = m_baselineIdentity; + } + + Json::StreamWriterBuilder writer; + // The value is stored as a single line in the sources settings file. + writer["indentation"] = ""; + + return Json::writeString(writer, root); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.h new file mode 100644 index 0000000000..f3d9f1541d --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.h @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once + +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + // The identities of the packages that make up a pre-indexed source, as carried by + // SourceDetails::Data. + // + // Data held exactly one package family name for as long as a source had exactly one package. + // A delta capable source uses a delta and the baseline that it names, each published under an + // identity of its own. Each identity must be known before acquiring that package, because a + // package is validated against the identity it was asked for. + // + // Two syntaxes are therefore accepted: + // + // Microsoft.Winget.Source_8wekyb3d8bbwe + // {"fullIdentity":"Microsoft.Winget.Source_8wekyb3d8bbwe"} + // {"deltaIdentity":"...","baselineIdentity":"..."} + // {"fullIdentity":"...","deltaIdentity":"...","baselineIdentity":"..."} + // + // The first is what every client written to date has stored, and remains what is written back + // for a source with no delta. The two are told apart by the leading brace, which a package + // family name cannot contain. + // + // Adding a delta capable source records all three identities, even though it acquires only + // the delta and baseline. The full identity lets the source remain usable when delta is + // disabled, without needing to discover an identity during an update or open. + // + // Enabling delta does not migrate a source whose Data names only a full index identity. It + // keeps using the full index, and a third party source that begins publishing a delta can be + // re-added. + struct SourceData + { + SourceData() = default; + + // Parses a SourceDetails::Data value in either syntax. + // Throws if the value is structured but malformed; an unparsable value is not silently + // treated as a package family name. + explicit SourceData(std::string_view data); + + // The identity of the full index, which is the only package a source that is not delta + // capable has. + const std::string& FullIdentity() const { return m_fullIdentity; } + void FullIdentity(std::string value) { m_fullIdentity = std::move(value); } + + // The identity of the delta, when the source publishes one. + const std::string& DeltaIdentity() const { return m_deltaIdentity; } + void DeltaIdentity(std::string value) { m_deltaIdentity = std::move(value); } + + // The identity of the baseline that the delta names. + // + // The baseline is published under an identity of its own rather than sharing the full + // index's. Sharing made the two indistinguishable to anything that holds packages by + // identity, so a client could not hold both, and the baseline had to be allowed to move + // backward over a full index that was merely newer. + const std::string& BaselineIdentity() const { return m_baselineIdentity; } + void BaselineIdentity(std::string value) { m_baselineIdentity = std::move(value); } + + bool HasFullIdentity() const { return !m_fullIdentity.empty(); } + bool HasDeltaIdentity() const { return !m_deltaIdentity.empty(); } + bool HasBaselineIdentity() const { return !m_baselineIdentity.empty(); } + + // The identity that the source itself is known by, as distinct from that of any one of + // its packages. + // + // The local state directory and the cross process lock are both derived from this, so it + // has to be settled when the source is added and stay fixed for as long as the source + // exists. The full index identity is preferred because that is what every source added + // before deltas existed is already keyed on; a source that has no full index is keyed on + // its delta, which is the package published at a fixed name and so the one that cannot + // move out from under it. + const std::string& SourceIdentity() const { return m_fullIdentity.empty() ? m_deltaIdentity : m_fullIdentity; } + + // Produces the value to store back into SourceDetails::Data. + // + // A source with only a full index serializes back to the bare family name it was read + // from, so that the stored value does not churn for the sources that are not delta + // capable. + std::string Serialize() const; + + private: + std::string m_fullIdentity; + std::string m_deltaIdentity; + std::string m_baselineIdentity; + }; + +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index 6af448b163..85d79bc6e2 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -4,10 +4,12 @@ #include "Microsoft/PreIndexedPackageSourceFactory.h" #include "Microsoft/PreIndexed/IndexForm.h" #include "Microsoft/PreIndexed/PackageStore.h" +#include "Microsoft/PreIndexed/SourceData.h" #include "Microsoft/SQLiteIndexSource.h" #include "SourceUpdateChecks.h" #include +#include using namespace std::string_literals; using namespace std::string_view_literals; @@ -17,6 +19,30 @@ namespace AppInstaller::Repository::Microsoft { namespace { + // Whether a source may be composed of a delta and a baseline rather than a full index. + bool IsDeltaIndexEnabled() + { + return Settings::ExperimentalFeature::IsEnabled(Settings::ExperimentalFeature::Feature::DeltaIndex); + } + + // Selects the form to use when opening a source. + std::unique_ptr SelectFormForOpen(const SourceDetails& details) + { + if (IsDeltaIndexEnabled()) + { + auto deltaForm = CreateDeltaIndexForm(details); + + // While experimental, lock on to delta if we have the bare minimum. + // Falling back to the full form is quite complex and may result in hiding issues. + if (deltaForm->HasIdentities()) + { + return deltaForm; + } + } + + return CreateFullIndexForm(details); + } + // Measures how long opening a source took, and which path got there. struct SourceOpenTimer { @@ -50,8 +76,10 @@ namespace AppInstaller::Repository::Microsoft { // We didn't use to store the source identifier, so we compute it here in case // it's missing from the details. - m_details.Identifier = m_details.Data; + m_details.Identifier = SourceData{ m_details.Data }.SourceIdentity(); } + + m_form = SelectFormForOpen(m_details); } std::string GetIdentifier() override { return m_details.Identifier; } @@ -60,9 +88,12 @@ namespace AppInstaller::Repository::Microsoft bool ShouldUpdateBeforeOpen(const std::optional& requestedUpdateInterval) override { + // Nothing here reports progress, but deciding whether the source holds usable data + // can require reading what it holds. + ProgressCallback progress; + auto store = CreateCompositeStore(m_details); - auto form = CreateFullIndexForm(m_details); - auto currentVersion = form->GetHeldVersion(*store); + auto currentVersion = m_form->GetHeldVersion(*store); // If we can't find a good package, then we have to update to operate if (!currentVersion) @@ -71,6 +102,13 @@ namespace AppInstaller::Repository::Microsoft return true; } + // Check that the form is consistent with itself + if (!m_form->IsUsable(*store, progress)) + { + AICLI_LOG(Repo, Verbose, << "Source `" << m_details.Name << "` is not currently usable"); + return true; + } + using namespace std::chrono_literals; using clock = std::chrono::system_clock; @@ -95,7 +133,6 @@ namespace AppInstaller::Repository::Microsoft SourceOpenTimer openTimer{ m_details.Name }; auto store = CreateCompositeStore(m_details); - auto form = CreateFullIndexForm(m_details); std::optional index; @@ -105,7 +142,7 @@ namespace AppInstaller::Repository::Microsoft // retried under it only when that fails. try { - index.emplace(form->Open(*store, progress)); + index.emplace(m_form->Open(*store, progress)); } catch (...) { @@ -128,7 +165,7 @@ namespace AppInstaller::Repository::Microsoft return {}; } - index.emplace(form->Open(*store, progress)); + index.emplace(m_form->Open(*store, progress)); } if (progress.IsCancelledBy(CancelReason::Any)) @@ -143,6 +180,7 @@ namespace AppInstaller::Repository::Microsoft private: SourceDetails m_details; + std::unique_ptr m_form; }; // The factory for a preindexed package source. @@ -177,15 +215,71 @@ namespace AppInstaller::Repository::Microsoft // Adding is the only operation that can establish what a source's packages are, so // it is the only one that discovers them and writes the result back. UpdateReport report; + UpdateResult result = UpdateResult::Unusable; - auto form = CreateFullIndexForm(details); - auto data = form->DiscoverIdentities(progress); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING, !data); + if (IsDeltaIndexEnabled()) + { + try + { + auto form = CreateDeltaIndexForm(details); + auto data = form->DiscoverIdentities(progress); - WriteDiscoveredIdentities(details, data.value()); + if (data) + { + WriteDiscoveredIdentities(details, data.value()); + + auto store = CreateStore(details); + result = form->Update(*store, false, progress, report); + + if (report.LearnedIdentities) + { + // The update learned the baseline identity. The full identity + // that the source is keyed on was already known. + WriteDiscoveredIdentities(details, report.LearnedIdentities.value()); + } + + if (result != UpdateResult::Success) + { + // The source is about to settle on the full index instead, and a + // form that was given up on is not something the source will ever + // come back to. Acquiring the full index would not displace the + // delta and baseline packages, so discard them first. + DiscardPackagesOf(details, *store, *form, progress); + } + } + } + catch (...) + { + if (progress.IsCancelledBy(CancelReason::Any)) + { + throw; + } - auto store = CreateStore(details); - UpdateResult result = form->Update(*store, false, progress, report); + LOG_CAUGHT_EXCEPTION_MSG("Delta discovery failed while adding source: %hs", details.Name.c_str()); + result = UpdateResult::Unusable; + } + } + + if (result == UpdateResult::Unusable) + { + // Either the source publishes no delta, or the one it publishes cannot serve + // it. Adding must still succeed, so the full index is acquired instead. + report = {}; + + // Nothing of the delta attempt is kept: the source is settling on the full + // index, and Data naming a delta is what would send it back to the other form. + details.Data.clear(); + details.Identifier.clear(); + + auto form = CreateFullIndexForm(details); + auto data = form->DiscoverIdentities(progress); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING, !data); + + WriteDiscoveredIdentities(details, data.value()); + + auto store = CreateStore(details); + result = form->Update(*store, false, progress, report); + } // Unlike an update, an add reports only when something was transferred. if (report.DownloadedBytes && report.NewIndexPublishedAt) @@ -231,13 +325,29 @@ namespace AppInstaller::Repository::Microsoft return false; } - auto form = CreateFullIndexForm(details); - + // Every form that the details name packages for contributes, so that a delta + // capable source does not leave its delta behind. std::vector packages; - if (form->HasIdentities()) - { - packages = form->GetPackages(); - } + std::set seenIdentities; + + auto addPackagesFrom = [&](const std::unique_ptr& form) + { + if (!form->HasIdentities()) + { + return; + } + + for (auto& package : form->GetPackages()) + { + if (seenIdentities.insert(package.Identity).second) + { + packages.emplace_back(std::move(package)); + } + } + }; + + addPackagesFrom(CreateFullIndexForm(details)); + addPackagesFrom(CreateDeltaIndexForm(details)); store->Remove(packages, progress); @@ -249,14 +359,87 @@ namespace AppInstaller::Repository::Microsoft static void WriteDiscoveredIdentities(SourceDetails& details, const std::string& data) { details.Data = data; - details.Identifier = data; + + // The identity of the source itself is derived from what it is made of rather + // than chosen, so that it is the same every time it is computed. The local state + // directory and the cross process lock are both derived from it, which is why it + // must not depend on how much has been learned so far; see + // SourceData::SourceIdentity. + details.Identifier = SourceData{ data }.SourceIdentity(); + } + + // Drops everything a form left behind, for a form that the source is not going to use. + // + // Best effort: the source is still going to be added, and leaving a package behind is + // not a reason to refuse it. + static void DiscardPackagesOf(const SourceDetails& details, IPackageStore& store, IIndexForm& form, IProgressCallback& progress) + { + try + { + auto lock = store.Lock(progress); + if (!lock) + { + return; + } + + // Only the failed form's packages have been acquired at this point. + store.Remove(form.GetPackages(), progress); + } + catch (...) + { + LOG_CAUGHT_EXCEPTION_MSG("Could not discard the packages of an unused form for source: %hs", details.Name.c_str()); + } } bool UpdateBase(const SourceDetails& details, bool isBackground, IProgressCallback& progress) + { THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); auto store = CreateStore(details); + + if (IsDeltaIndexEnabled()) + { + UpdateReport report; + UpdateResult result = UpdateResult::Unusable; + + auto form = CreateDeltaIndexForm(details); + + // An update never probes for a delta: the details say whether the source + // has one. A source configured with only a full index identity names no + // delta and uses the full index, which is what it did before deltas existed. + if (form->HasIdentities()) + { + result = form->Update(*store, isBackground, progress, report); + + // While the feature is experimental, fail if things are not as they are expected + // When moving off of experimental, consider if we should support fallback. + // Fallback complicates the open side considerably. + if (IsDeltaIndexEnabled()) + { + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, result == UpdateResult::Unusable); + } + } + + if (result == UpdateResult::Success) + { + LogUpdate(details, report, isBackground); + return true; + } + + if (result == UpdateResult::Aborted) + { + // The operation did not complete, so nothing was learned about whether the + // delta would have served. Falling back here would acquire a full index + // that the source may not have needed. + return false; + } + + // Only a delta that cannot serve the source falls back; the delta is an + // optimization, never a correctness dependency. + AICLI_LOG(Repo, Info, << "Falling back to the full index for source: " << details.Name); + } + auto form = CreateFullIndexForm(details); UpdateReport report; diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp index c5ce7d6867..e371274294 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -45,7 +45,12 @@ namespace AppInstaller::Repository::Microsoft return { filePath, source }; } - SQLiteIndex SQLiteIndex::OpenWithBaseline(const std::string& deltaFilePath, const std::string& baselineFilePath, OpenDisposition disposition) + SQLiteIndex SQLiteIndex::OpenWithBaseline( + const std::string& deltaFilePath, + const std::string& baselineFilePath, + OpenDisposition disposition, + Utility::ManagedFile&& deltaFile, + Utility::ManagedFile&& baselineFile) { AICLI_LOG(Repo, Info, << "Opening delta index [" << deltaFilePath << "] with baseline [" << baselineFilePath << "]"); @@ -56,7 +61,9 @@ namespace AppInstaller::Repository::Microsoft std::filesystem::path baselinePath{ Utility::ConvertToUTF16(baselineFilePath) }; THROW_HR_IF(E_INVALIDARG, baselinePath.empty() || baselinePath.is_relative()); - SQLiteIndex result{ SQLite::DatabaseSpecifier{ deltaFilePath, disposition }, {} }; + SQLiteIndex result{ + SQLite::DatabaseSpecifier{ deltaFilePath, disposition }, + InitializeFromMoveOnly>({ std::move(deltaFile), std::move(baselineFile) }) }; // The interface for the delta's schema version establishes the combined view. result.m_interface->SetupDeltaReadMode(result.m_dbconn, SQLite::DatabaseSpecifier{ baselineFilePath, disposition }); @@ -64,6 +71,18 @@ namespace AppInstaller::Repository::Microsoft return result; } + std::optional SQLiteIndex::GetDeltaBaselineLocator() const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return m_interface->GetDeltaBaselineLocator(m_dbconn); + } + + std::optional SQLiteIndex::GetBaselineIdentifier() const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return m_interface->GetBaselineIdentifier(m_dbconn); + } + SQLiteIndex::SQLiteIndex(const std::string& target, const SQLite::Version& version, CreateOptions options) : SQLiteStorageBase(target, version, GetPageSizeFromOptions(options)) { m_dbconn.EnableICU(); @@ -73,12 +92,12 @@ namespace AppInstaller::Repository::Microsoft } SQLiteIndex::SQLiteIndex(const std::string& target, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile) : - SQLiteIndex(SQLite::DatabaseSpecifier{ target, disposition }, std::move(indexFile)) + SQLiteIndex(SQLite::DatabaseSpecifier{ target, disposition }, InitializeFromMoveOnly>({ std::move(indexFile) })) { } - SQLiteIndex::SQLiteIndex(const SQLite::DatabaseSpecifier& specifier, Utility::ManagedFile&& indexFile) : - SQLiteStorageBase(specifier, std::move(indexFile)) + SQLiteIndex::SQLiteIndex(const SQLite::DatabaseSpecifier& specifier, std::vector&& files) : + SQLiteStorageBase(specifier, std::move(files)) { m_dbconn.EnableICU(); AICLI_LOG(Repo, Info, << "Opened SQLite Index with version [" << m_version << "], last write [" << GetLastWriteTime() << "]"); diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h index e8c84af14b..7cfb7fc5b9 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h @@ -59,7 +59,29 @@ namespace AppInstaller::Repository::Microsoft // The delta is the main connection; the baseline is attached and temp views are created // so that existing search code operates transparently across both. // The disposition applies to both files, because the pair is only meaningful as a unit. - static SQLiteIndex OpenWithBaseline(const std::string& deltaFilePath, const std::string& baselineFilePath, OpenDisposition disposition = OpenDisposition::Read); + // Both files may be given to be held open for the lifetime of the index, as the caller may + // have extracted either of them to a temporary location. + static SQLiteIndex OpenWithBaseline( + const std::string& deltaFilePath, + const std::string& baselineFilePath, + OpenDisposition disposition = OpenDisposition::Read, + Utility::ManagedFile&& deltaFile = {}, + Utility::ManagedFile&& baselineFile = {}); + + // What a delta records about the baseline that it must be paired with. + using DeltaBaselineLocator = Schema::ISQLiteIndex::DeltaBaselineLocator; + + // Gets what this delta records about the baseline that it must be paired with. + // + // Returns nothing when this is not a delta, or does not fully name a baseline. Safe to + // ask of any index, including one whose version predates deltas. + std::optional GetDeltaBaselineLocator() const; + + // Gets the identifier that designates this index as a baseline, if it carries one. + // + // Returns nothing for any index that has not been designated, which is every index not + // prepared with DeltaMarkAsBaseline. Such an index cannot be paired with a delta. + std::optional GetBaselineIdentifier() const; #ifndef AICLI_DISABLE_TEST_HOOKS // Changes the version of the interface being used to operate on the database. @@ -199,7 +221,7 @@ namespace AppInstaller::Repository::Microsoft // Constructor used to open an existing index. SQLiteIndex(const std::string& target, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile); - SQLiteIndex(const SQLite::DatabaseSpecifier& specifier, Utility::ManagedFile&& indexFile); + SQLiteIndex(const SQLite::DatabaseSpecifier& specifier, std::vector&& files); // Constructor used to copy the given index. SQLiteIndex(const std::string& target, SQLiteIndex& source); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp index 4d7e9b7925..657d61ae72 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp @@ -297,10 +297,52 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta } } + std::optional ReadBaselineLocator(const SQLite::Connection& connection) + { + if (!IsDeltaDatabase(connection)) + { + return std::nullopt; + } + + std::optional identifier = SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_DeltaBaselineIdentifier); + std::optional relativeSourcePath = SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_DeltaBaselineRelativeSourcePath); + std::optional packageVersion = SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_DeltaBaselinePackageVersion); + + // Generation writes all three together and requires each to be non-empty, so anything less + // than the full set means the delta was damaged or was not produced by us. Either way there + // is no baseline that it can be safely paired with. + if (!identifier || identifier->empty() || + !relativeSourcePath || relativeSourcePath->empty() || + !packageVersion || packageVersion->empty()) + { + AICLI_LOG(Repo, Warning, << "Delta does not fully name its baseline; it cannot be paired with one"); + return std::nullopt; + } + + BaselineLocator result; + result.Identifier = std::move(identifier).value(); + result.RelativeSourcePath = std::move(relativeSourcePath).value(); + result.PackageVersion = std::move(packageVersion).value(); + return result; + } + + std::optional ReadBaselineIdentifier(const SQLite::Connection& connection) + { + std::optional result = SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_BaselineIdentifier); + + if (result && result->empty()) + { + return std::nullopt; + } + + return result; + } + void Generate( const SQLite::Connection& sourceConnection, const SQLite::Connection& baselineConnection, - const BaselineReference& baselineReference, + const std::string& baselineRelativeSourcePath, + const std::string& baselinePackageVersion, const std::filesystem::path& deltaOutputPath, const SQLite::Version& version, const std::vector& changedPackages, @@ -310,7 +352,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta " changed and " << removedPackages.size() << " removed packages"); // A client that holds only the delta has to be able to find the baseline, so it must be named. - THROW_HR_IF(E_INVALIDARG, baselineReference.RelativeSourcePath.empty() || baselineReference.PackageVersion.empty()); + THROW_HR_IF(E_INVALIDARG, baselineRelativeSourcePath.empty() || baselinePackageVersion.empty()); // A delta is only meaningful alongside the exact baseline it was computed from, so the // baseline has to be one that was designated as such and can therefore be named. @@ -365,8 +407,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta SQLite::MetadataTable::GetNamedValue(sourceConnection, SQLite::s_MetadataValueName_DatabaseIdentifier)); SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselineIdentifier, baselineIdentifier.value()); - SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselineRelativeSourcePath, baselineReference.RelativeSourcePath); - SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselinePackageVersion, baselineReference.PackageVersion); + SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselineRelativeSourcePath, baselineRelativeSourcePath); + SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselinePackageVersion, baselinePackageVersion); std::map nextValueRowIds; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h index d63795032a..96e1f34f75 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. #pragma once +#include "Microsoft/Schema/ISQLiteIndex.h" #include "Microsoft/Schema/2_0/PackageUpdateTrackingTable.h" #include #include #include +#include #include #include #include @@ -12,17 +14,23 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta { - // Identifies the baseline package to a client that holds only the delta, so that it can acquire - // the baseline the delta was generated against. Both values are supplied by the publishing - // service, which is the only party that knows how its baselines are laid out and versioned. - struct BaselineReference - { - // The location of the baseline package, relative to the source's base location. - std::string RelativeSourcePath; + // Defined on ISQLiteIndex, because it crosses from here out to the business logic and that is + // the only header both sides share. It is named here without the Delta prefix that + // disambiguates it there, since this namespace already supplies it. + using BaselineLocator = ISQLiteIndex::DeltaBaselineLocator; - // The version of the baseline package. - std::string PackageVersion; - }; + // Reads what a delta records about its baseline. + // + // Returns nothing when the database is not a delta, or when it does not carry all three + // values. A delta that cannot fully name its baseline is unusable, and reporting a partial + // locator would only invite a caller to act on half of one. + std::optional ReadBaselineLocator(const SQLite::Connection& connection); + + // Reads the identifier that designates a database as a baseline, if it carries one. + // + // Returns nothing for any index that has not been designated, which is every index that was + // not prepared with DeltaMarkAsBaseline. Such an index cannot be paired with a delta. + std::optional ReadBaselineIdentifier(const SQLite::Connection& connection); // Writes a delta database describing the difference between a baseline index and the index // that is currently being packaged. @@ -40,10 +48,16 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta // Nothing appears at the output path until the delta is complete: it is built beside the // destination and moved into place only on success, so a failure partway through cannot leave // something that looks like a usable delta. An output path that already exists is refused. + // + // The two baseline values are the caller's half of the locator that the delta records: where + // the baseline will be published and which version it is. The third, the baseline's identity, + // is not the caller's to supply -- it is read from the baseline itself, which is also what + // establishes that the baseline was designated as one at all. void Generate( const SQLite::Connection& sourceConnection, const SQLite::Connection& baselineConnection, - const BaselineReference& baselineReference, + const std::string& baselineRelativeSourcePath, + const std::string& baselinePackageVersion, const std::filesystem::path& deltaOutputPath, const SQLite::Version& version, const std::vector& changedPackages, diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h index 3bc372ac83..a8b0493fee 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h @@ -48,6 +48,14 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1 // read path operates on the combination. Must be called before any read. void SetupDeltaReadMode(SQLite::Connection& connection, const SQLite::DatabaseSpecifier& baseline) override; + // Gets what this delta records about the baseline it must be paired with, or nothing when + // this is not a delta or does not fully name one. + std::optional GetDeltaBaselineLocator(const SQLite::Connection& connection) const override; + + // Gets the identifier designating this index as a baseline, or nothing when it has not + // been designated. + std::optional GetBaselineIdentifier(const SQLite::Connection& connection) const override; + protected: // Records the baseline sequence for this index, and generates a delta index against a previous // baseline when the caller has supplied the paths to do so. diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp index 17249f376c..7a33c21ff6 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp @@ -82,6 +82,16 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1 m_internalInterfaceChecked = true; } + std::optional Interface::GetDeltaBaselineLocator(const SQLite::Connection& connection) const + { + return Delta::ReadBaselineLocator(connection); + } + + std::optional Interface::GetBaselineIdentifier(const SQLite::Connection& connection) const + { + return Delta::ReadBaselineIdentifier(connection); + } + bool Interface::CheckConsistency(const SQLiteIndexConstContext& context, bool log) const { bool hasBaseline = context.Data.Contains(Property::DeltaBaselineIndexPath); @@ -187,18 +197,17 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1 std::filesystem::path deltaOutputPath = context.Data.Get(); - Delta::BaselineReference baselineReference - { - context.Data.Get(), - context.Data.Get(), - }; + // The caller's half of the locator that the delta will record; the baseline's identity is + // read from the baseline itself by generation. + const std::string& baselineRelativeSourcePath = context.Data.Get(); + const std::string& baselinePackageVersion = context.Data.Get(); if (markAsBaseline) { MarkAsBaseline(connection); // This index is its own baseline, so nothing has changed since it and the delta that describes it is empty. - Delta::Generate(connection, connection, baselineReference, deltaOutputPath, GetVersion(), {}, {}); + Delta::Generate(connection, connection, baselineRelativeSourcePath, baselinePackageVersion, deltaOutputPath, GetVersion(), {}, {}); return; } @@ -230,7 +239,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1 Delta::Generate( connection, baselineConnection, - baselineReference, + baselineRelativeSourcePath, + baselinePackageVersion, deltaOutputPath, GetVersion(), changedPackages, diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp index 01c6c7a3f6..709e77d978 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp @@ -39,6 +39,18 @@ namespace AppInstaller::Repository::Microsoft::Schema THROW_WIN32(ERROR_NOT_SUPPORTED); } + std::optional ISQLiteIndex::GetDeltaBaselineLocator(const SQLite::Connection&) const + { + // A version that predates deltas cannot be one. + return std::nullopt; + } + + std::optional ISQLiteIndex::GetBaselineIdentifier(const SQLite::Connection&) const + { + // A version that predates deltas cannot have been designated as a baseline. + return std::nullopt; + } + std::unique_ptr CreateISQLiteIndex(const SQLite::Version& version) { if (version.MajorVersion == 1 || diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h index 42c568d551..93313d6661 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h @@ -169,10 +169,47 @@ namespace AppInstaller::Repository::Microsoft::Schema // Version 2.1 + // Everything that a delta records about the baseline it must be paired with, which is all + // a client holding only the delta needs in order to acquire the baseline. + // + // The location and version are supplied by the publishing service, which is the only party + // that knows how its baselines are laid out and versioned; the identifier is read from the + // baseline itself during generation. + struct DeltaBaselineLocator + { + // The baseline identifier that the delta names. A baseline acquired for this delta + // must carry the same value, or the two were not generated as a pair. + std::string Identifier; + + // The location of the baseline package, relative to the source's base location. + std::string RelativeSourcePath; + + // The version of the baseline package. + std::string PackageVersion; + }; + // Sets this index up to read the combination of a delta and the baseline that it was // generated against, so that every subsequent read sees the merged data. Must be called // before any read. virtual void SetupDeltaReadMode(SQLite::Connection& connection, const SQLite::DatabaseSpecifier& baseline); + + // Gets what this delta records about the baseline that it must be paired with. + // + // Returns nothing when this is not a delta, or when it does not carry all three values. + // A delta that cannot fully name its baseline is unusable, and reporting a partial locator + // would only invite a caller to act on half of one. + // + // Unlike the operations above this does not throw for a version that predates deltas: + // such an index is definitively not a delta, so nothing is the true answer rather than a + // refusal, and a caller can ask without first establishing the version. + virtual std::optional GetDeltaBaselineLocator(const SQLite::Connection& connection) const; + + // Gets the identifier that designates this index as a baseline, if it carries one. + // + // Returns nothing for any index that has not been designated, which is every index that + // was not prepared with DeltaMarkAsBaseline, and every version that predates deltas. Such + // an index cannot be paired with a delta. + virtual std::optional GetBaselineIdentifier(const SQLite::Connection& connection) const; }; DEFINE_ENUM_FLAG_OPERATORS(ISQLiteIndex::CreateOptions); diff --git a/src/AppInstallerRepositoryCore/SourceList.cpp b/src/AppInstallerRepositoryCore/SourceList.cpp index bffbeeef57..359a35e928 100644 --- a/src/AppInstallerRepositoryCore/SourceList.cpp +++ b/src/AppInstallerRepositoryCore/SourceList.cpp @@ -4,6 +4,7 @@ #include "SourceList.h" #include "SourcePolicy.h" #include "Microsoft/PreIndexedPackageSourceFactory.h" +#include "Microsoft/PreIndexed/SourceData.h" #include "Rest/RestSourceFactory.h" #include @@ -38,9 +39,28 @@ namespace AppInstaller::Repository constexpr std::string_view s_Source_WingetCommunityDefault_Name = "winget"sv; constexpr std::string_view s_Source_WingetCommunityDefault_Arg = "https://cdn.winget.microsoft.com/cache"sv; - constexpr std::string_view s_Source_WingetCommunityDefault_Data = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; + constexpr std::string_view s_Source_WingetCommunityDefault_FullIdentity = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; + constexpr std::string_view s_Source_WingetCommunityDefault_DeltaIdentity = "Microsoft.Winget.Source.Delta_8wekyb3d8bbwe"sv; + constexpr std::string_view s_Source_WingetCommunityDefault_BaselineIdentity = "Microsoft.Winget.Source.Baseline_8wekyb3d8bbwe"sv; constexpr std::string_view s_Source_WingetCommunityDefault_Identifier = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; + // The Data of the built in winget source, which names its delta and baseline as well as + // its full index. + // + // This is the one source that names all three. A source that is added settles on one form + // or the other, but this one is declared rather than discovered, and naming the full index + // is what keeps it known by the identity it has always had: the local state directory and + // the cross process lock are derived from that, and every machine the client has ever run + // on already has state under it. + std::string GetWingetCommunityDefaultData() + { + Microsoft::PreIndexed::SourceData data; + data.FullIdentity(std::string{ s_Source_WingetCommunityDefault_FullIdentity }); + data.DeltaIdentity(std::string{ s_Source_WingetCommunityDefault_DeltaIdentity }); + data.BaselineIdentity(std::string{ s_Source_WingetCommunityDefault_BaselineIdentity }); + return data.Serialize(); + } + constexpr std::string_view s_Source_MSStoreDefault_Name = "msstore"sv; constexpr std::string_view s_Source_MSStoreDefault_Arg = "https://storeedgefd.dsx.mp.microsoft.com/v9.0"sv; constexpr std::string_view s_Source_MSStoreDefault_Identifier = "StoreEdgeFD"sv; @@ -348,7 +368,7 @@ namespace AppInstaller::Repository details.Name = s_Source_WingetCommunityDefault_Name; details.Type = Microsoft::PreIndexedPackageSourceFactory::Type(); details.Arg = s_Source_WingetCommunityDefault_Arg; - details.Data = s_Source_WingetCommunityDefault_Data; + details.Data = GetWingetCommunityDefaultData(); details.Identifier = s_Source_WingetCommunityDefault_Identifier; details.TrustLevel = SourceTrustLevel::Trusted | SourceTrustLevel::StoreOrigin; return details; diff --git a/src/AppInstallerSharedLib/Public/AppInstallerLanguageUtilities.h b/src/AppInstallerSharedLib/Public/AppInstallerLanguageUtilities.h index 505a4eb00d..8f01ade1c3 100644 --- a/src/AppInstallerSharedLib/Public/AppInstallerLanguageUtilities.h +++ b/src/AppInstallerSharedLib/Public/AppInstallerLanguageUtilities.h @@ -16,6 +16,20 @@ namespace AppInstaller { + template + Container InitializeFromMoveOnly(std::initializer_list il) + { + using Value = typename Container::value_type; + Container result; + + for (const auto& v : il) + { + result.emplace_back(std::move(*const_cast(&v))); + } + + return result; + } + // A helper type that resets itself when it is moved from. template struct ResetWhenMovedFrom diff --git a/src/AppInstallerSharedLib/Public/winget/SQLiteStorageBase.h b/src/AppInstallerSharedLib/Public/winget/SQLiteStorageBase.h index 4b8a659d2d..cfc04b5b1e 100644 --- a/src/AppInstallerSharedLib/Public/winget/SQLiteStorageBase.h +++ b/src/AppInstallerSharedLib/Public/winget/SQLiteStorageBase.h @@ -7,6 +7,7 @@ #include #include +#include namespace AppInstaller::SQLite { @@ -16,6 +17,9 @@ namespace AppInstaller::SQLite // The disposition for opening the database. using OpenDisposition = DatabaseDisposition; + SQLiteStorageBase(SQLiteStorageBase&&) = default; + SQLiteStorageBase& operator=(SQLiteStorageBase&& other) noexcept; + // Gets the last write time for the database. std::chrono::system_clock::time_point GetLastWriteTime() const; @@ -37,12 +41,15 @@ namespace AppInstaller::SQLite SQLiteStorageBase(const DatabaseSpecifier& specifier, Utility::ManagedFile&& indexFile); + SQLiteStorageBase(const DatabaseSpecifier& specifier, std::vector&& files); + SQLiteStorageBase(const std::string& target, SQLiteStorageBase& source); // Sets the last write time metadata value in the database. void SetLastWriteTime(); - Utility::ManagedFile m_indexFile; + // Keep every backing file alive until after the connection (including attachments) closes. + std::vector m_files; SQLite::Connection m_dbconn; Version m_version; std::unique_ptr m_interfaceLock = std::make_unique(); diff --git a/src/AppInstallerSharedLib/Runtime.cpp b/src/AppInstallerSharedLib/Runtime.cpp index de48562489..e73927ecb5 100644 --- a/src/AppInstallerSharedLib/Runtime.cpp +++ b/src/AppInstallerSharedLib/Runtime.cpp @@ -214,8 +214,23 @@ namespace AppInstaller::Runtime return IsRunningAsAdmin() || IsRunningAsSystem(); } +#ifndef AICLI_DISABLE_TEST_HOOKS + static bool* s_IsRunningAsInteractiveUser_TestHook_Override = nullptr; + + void TestHook_SetIsRunningAsInteractiveUser_Override(bool* value) + { + s_IsRunningAsInteractiveUser_TestHook_Override = value; + } +#endif + bool IsRunningAsInteractiveUser() { +#ifndef AICLI_DISABLE_TEST_HOOKS + if (s_IsRunningAsInteractiveUser_TestHook_Override) + { + return *s_IsRunningAsInteractiveUser_TestHook_Override; + } +#endif return wil::test_token_membership(nullptr, SECURITY_NT_AUTHORITY, SECURITY_INTERACTIVE_RID); } diff --git a/src/AppInstallerSharedLib/SQLiteStorageBase.cpp b/src/AppInstallerSharedLib/SQLiteStorageBase.cpp index dff989519f..50d5e5e60a 100644 --- a/src/AppInstallerSharedLib/SQLiteStorageBase.cpp +++ b/src/AppInstallerSharedLib/SQLiteStorageBase.cpp @@ -4,6 +4,7 @@ #include "Public/winget/SQLiteStorageBase.h" #include "Public/winget/SQLiteMetadataTable.h" #include "AppInstallerDateTime.h" +#include "AppInstallerLanguageUtilities.h" namespace AppInstaller::SQLite { @@ -39,6 +40,20 @@ namespace AppInstaller::SQLite } } + SQLiteStorageBase& SQLiteStorageBase::operator=(SQLiteStorageBase&& other) noexcept + { + if (this != &other) + { + // Release the old connection before deleting any of its backing files. + m_dbconn = std::move(other.m_dbconn); + m_files = std::move(other.m_files); + m_version = std::move(other.m_version); + m_interfaceLock = std::move(other.m_interfaceLock); + } + + return *this; + } + void SQLiteStorageBase::SetLastWriteTime() { MetadataTable::SetNamedValue(m_dbconn, s_MetadataValueName_LastWriteTime, Utility::GetCurrentUnixEpoch()); @@ -94,7 +109,12 @@ namespace AppInstaller::SQLite } SQLiteStorageBase::SQLiteStorageBase(const DatabaseSpecifier& specifier, Utility::ManagedFile&& file) : - m_indexFile(std::move(file)) + SQLiteStorageBase(specifier, InitializeFromMoveOnly>({ std::move(file) })) + { + } + + SQLiteStorageBase::SQLiteStorageBase(const DatabaseSpecifier& specifier, std::vector&& files) : + m_files(std::move(files)) { AICLI_LOG(Repo, Info, << "Opening database for " << GetOpenDispositionString(specifier.Disposition()) << " at '" << specifier.Path() << "'"); diff --git a/src/WinGetSourceCreator/Helpers.cs b/src/WinGetSourceCreator/Helpers.cs index 709e49ba78..f3ae7a8aa6 100644 --- a/src/WinGetSourceCreator/Helpers.cs +++ b/src/WinGetSourceCreator/Helpers.cs @@ -141,7 +141,7 @@ public static void RunCommand(string command, string args, string? workingDirect } // If in the future we edit more elements, this should be a nice wrapper class. - public static void ModifyAppxManifestIdentity(string manifestFile, string? identityPublisher) + public static void ModifyAppxManifestIdentity(string manifestFile, string? identityPublisher, string? identityName = null, string? identityVersion = null) { if (!File.Exists(manifestFile)) { @@ -168,6 +168,26 @@ public static void ModifyAppxManifestIdentity(string manifestFile, string? ident attr.Value = identityPublisher; } + if (!string.IsNullOrEmpty(identityName)) + { + var attr = identityNode.Attributes?["Name"]; + if (attr == null) + { + throw new NullReferenceException("Name attribute"); + } + attr.Value = identityName; + } + + if (!string.IsNullOrEmpty(identityVersion)) + { + var attr = identityNode.Attributes?["Version"]; + if (attr == null) + { + throw new NullReferenceException("Version attribute"); + } + attr.Value = identityVersion; + } + xmlDoc.Save(manifestFile); } diff --git a/src/WinGetSourceCreator/Model/DeltaSettings.cs b/src/WinGetSourceCreator/Model/DeltaSettings.cs new file mode 100644 index 0000000000..045f321f1a --- /dev/null +++ b/src/WinGetSourceCreator/Model/DeltaSettings.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace WinGetSourceCreator.Model +{ + // Describes a delta capable source to publish alongside the ordinary one. + // + // It is published as its own source rather than layered onto the existing packages so that a + // client that knows nothing about deltas keeps seeing exactly what it saw before. + public class DeltaSettings + { + // Directory, relative to the working directory, that the delta source is published under. + // This is also what a client is pointed at, so everything the source offers lives below it. + public string RelativeDirectory { get; set; } = "deltaSource"; + + // Identity name carried by the full index. + public string IdentityName { get; set; } = string.Empty; + + // Identity name carried by the delta, which has to differ from the above so that a client + // can acquire one without the other. + public string DeltaIdentityName { get; set; } = string.Empty; + + // Identity name carried by the baseline. A baseline is a full index in content, but it is + // published as a package in its own right: a client on the delta holds a baseline and no + // full index, while one that is not holds the opposite, and a store that addresses packages + // by identity could not tell the two apart if they shared one. + public string BaselineIdentityName { get; set; } = string.Empty; + + // The version of the full index and of the delta that describes it. + public string Version { get; set; } = string.Empty; + + // The baseline is published at an older version than the full index, because a baseline is + // captured once and then described by a succession of deltas. The two no longer share an + // identity, so this is realism rather than a constraint the client depends on. + public string BaselineVersion { get; set; } = string.Empty; + + // Where the baseline is published, relative to the source's own location. The delta records + // this, so it is what a client will ask for. + public string BaselineRelativePath { get; set; } = "baseline/source2.msix"; + + // Manifest file names left out of the baseline. Without these the delta would be empty and + // a test could not tell a merged result apart from either half read on its own. + public List ManifestsNotInBaseline { get; set; } = new(); + + public void Validate() + { + if (string.IsNullOrEmpty(this.RelativeDirectory)) + { + throw new ArgumentNullException(nameof(this.RelativeDirectory)); + } + + if (string.IsNullOrEmpty(this.IdentityName)) + { + throw new ArgumentNullException(nameof(this.IdentityName)); + } + + if (string.IsNullOrEmpty(this.DeltaIdentityName)) + { + throw new ArgumentNullException(nameof(this.DeltaIdentityName)); + } + + if (string.IsNullOrEmpty(this.BaselineIdentityName)) + { + throw new ArgumentNullException(nameof(this.BaselineIdentityName)); + } + + // All three are addressed independently by a client, so any two of them sharing a name + // would make the source untestable rather than merely unrealistic. + var identities = new HashSet( + new[] { this.IdentityName, this.DeltaIdentityName, this.BaselineIdentityName }, + StringComparer.Ordinal); + + if (identities.Count != 3) + { + throw new ArgumentException("The full index, the delta and the baseline must each have an identity of their own", nameof(this.IdentityName)); + } + + if (string.IsNullOrEmpty(this.Version)) + { + throw new ArgumentNullException(nameof(this.Version)); + } + + if (string.IsNullOrEmpty(this.BaselineVersion)) + { + throw new ArgumentNullException(nameof(this.BaselineVersion)); + } + + if (string.IsNullOrEmpty(this.BaselineRelativePath)) + { + throw new ArgumentNullException(nameof(this.BaselineRelativePath)); + } + + if (this.ManifestsNotInBaseline.Count == 0) + { + throw new ArgumentException("An empty delta cannot demonstrate anything", nameof(this.ManifestsNotInBaseline)); + } + } + } +} diff --git a/src/WinGetSourceCreator/Model/LocalSource.cs b/src/WinGetSourceCreator/Model/LocalSource.cs index 79d8bd6a2a..ae1cb349cd 100644 --- a/src/WinGetSourceCreator/Model/LocalSource.cs +++ b/src/WinGetSourceCreator/Model/LocalSource.cs @@ -23,6 +23,9 @@ public class LocalSource public Signature? Signature { get; set; } + // When set, a baseline and a delta are published alongside the ordinary source. + public DeltaSettings? Delta { get; set; } + public void Validate() { if (string.IsNullOrEmpty(this.AppxManifest)) @@ -60,6 +63,11 @@ public void Validate() { this.Signature.Validate(); } + + if (this.Delta != null) + { + this.Delta.Validate(); + } } public string GetIndexName() diff --git a/src/WinGetSourceCreator/WinGetLocalSource.cs b/src/WinGetSourceCreator/WinGetLocalSource.cs index bff56efc31..8f4afd96ac 100644 --- a/src/WinGetSourceCreator/WinGetLocalSource.cs +++ b/src/WinGetSourceCreator/WinGetLocalSource.cs @@ -72,6 +72,135 @@ public static void CreateLocalSource(LocalSource localSource) var indexV1File = wingetSource.CreateIndex(localSource.GetIndexName()); _ = wingetSource.CreatePackage(localSource.GetSourceName(1), localSource.AppxManifest, indexV1File, localSource.Signature); + + if (localSource.Delta != null) + { + wingetSource.CreateDeltaSource(localSource, localSource.Delta); + } + } + + // Publishes a second, delta capable source: a baseline, the full index, and the delta that + // carries the difference between them. + // + // The baseline deliberately omits some manifests. If it held everything then the delta + // would be empty, and a client that silently ignored it would look exactly like one that + // merged it correctly. + public void CreateDeltaSource(LocalSource localSource, DeltaSettings delta) + { + // Where the packages are published, and so what the source's URL resolves to. Preparing + // an index also emits a per package version data file under this directory, which the + // client fetches relative to that URL for anything beyond a search. + string publishDirectory = Path.Combine(this.workingDirectory, delta.RelativeDirectory); + + // The working databases. They are never named by a package, but they do sit under the + // static file root and so are reachable; that is already true of the ordinary source's + // index.db and costs nothing here. + string indexDirectory = Path.Combine(this.workingDirectory, "delta_indexes"); + string workingIndexPath = Path.Combine(indexDirectory, "working.db"); + string baselineIndexPath = Path.Combine(indexDirectory, "baseline.db"); + string fullIndexPath = Path.Combine(indexDirectory, "full.db"); + string deltaIndexPath = Path.Combine(indexDirectory, "delta.db"); + + // Produced by designating the baseline, and of no use to anyone: an index that is its + // own baseline has nothing to say. + string discardedDeltaPath = Path.Combine(indexDirectory, "baseline.delta.db"); + + if (Directory.Exists(indexDirectory)) + { + Directory.Delete(indexDirectory, true); + } + + Directory.CreateDirectory(indexDirectory); + + var excluded = new HashSet(delta.ManifestsNotInBaseline, StringComparer.OrdinalIgnoreCase); + var allManifests = Directory.EnumerateFiles(this.workingDirectory, "*.yaml", SearchOption.AllDirectories).ToList(); + var baselineManifests = allManifests.Where(file => !excluded.Contains(Path.GetFileName(file))).ToList(); + + if (baselineManifests.Count == allManifests.Count) + { + throw new InvalidOperationException("None of the manifests named by ManifestsNotInBaseline were found, so the delta would be empty."); + } + + WinGetFactory factory = new (); + + // The baseline and the full index are both prepared from copies of one working index. + // That is not merely convenient: generation refuses a baseline whose database identifier + // is not the source's, and the delta's window is the change sequence that the baseline + // recorded, so two independently built indexes could not be paired at all. + using (IWinGetSQLiteIndex workingIndex = factory.SQLiteIndexCreate(workingIndexPath, 2, 1)) + { + this.AddManifestsToIndex(workingIndex, baselineManifests); + } + + File.Copy(workingIndexPath, baselineIndexPath); + using (IWinGetSQLiteIndex baselineIndex = factory.SQLiteIndexOpen(baselineIndexPath)) + { + baselineIndex.SetProperty(SQLiteIndexProperty.IntermediateFileOutputPath, publishDirectory); + baselineIndex.SetProperty(SQLiteIndexProperty.DeltaMarkAsBaseline, "true"); + baselineIndex.SetProperty(SQLiteIndexProperty.DeltaOutputPath, discardedDeltaPath); + baselineIndex.SetProperty(SQLiteIndexProperty.DeltaBaselineRelativeSourcePath, delta.BaselineRelativePath); + baselineIndex.SetProperty(SQLiteIndexProperty.DeltaBaselinePackageVersion, delta.BaselineVersion); + baselineIndex.PrepareForPackaging(); + } + + // The manifests the baseline was built without become the delta's content. + using (IWinGetSQLiteIndex workingIndex = factory.SQLiteIndexOpen(workingIndexPath)) + { + this.AddManifestsToIndex(workingIndex, allManifests.Except(baselineManifests)); + } + + File.Copy(workingIndexPath, fullIndexPath); + using (IWinGetSQLiteIndex fullIndex = factory.SQLiteIndexOpen(fullIndexPath)) + { + fullIndex.SetProperty(SQLiteIndexProperty.IntermediateFileOutputPath, publishDirectory); + fullIndex.SetProperty(SQLiteIndexProperty.DeltaBaselineIndexPath, baselineIndexPath); + fullIndex.SetProperty(SQLiteIndexProperty.DeltaOutputPath, deltaIndexPath); + fullIndex.SetProperty(SQLiteIndexProperty.DeltaBaselineRelativeSourcePath, delta.BaselineRelativePath); + fullIndex.SetProperty(SQLiteIndexProperty.DeltaBaselinePackageVersion, delta.BaselineVersion); + fullIndex.PrepareForPackaging(); + } + + if (!File.Exists(deltaIndexPath)) + { + throw new InvalidOperationException($"Preparing the full index did not produce a delta at {deltaIndexPath}"); + } + + // Every package names the index as "index.db" regardless of the working file it came + // from, since that is the only name a client looks for. + string indexNameInPackage = localSource.GetIndexName(); + + // The relative path is recorded in the delta as a URL fragment, so it keeps its forward + // slashes there; only the on disk location needs separators this machine understands. + string baselinePackagePath = Path.Combine( + delta.RelativeDirectory, + delta.BaselineRelativePath.Replace('/', Path.DirectorySeparatorChar)); + + _ = this.CreatePackage( + baselinePackagePath, + localSource.AppxManifest, + baselineIndexPath, + localSource.Signature, + delta.BaselineIdentityName, + delta.BaselineVersion, + indexNameInPackage); + + _ = this.CreatePackage( + Path.Combine(delta.RelativeDirectory, localSource.GetSourceName(2)), + localSource.AppxManifest, + fullIndexPath, + localSource.Signature, + delta.IdentityName, + delta.Version, + indexNameInPackage); + + _ = this.CreatePackage( + Path.Combine(delta.RelativeDirectory, "delta.msix"), + localSource.AppxManifest, + deltaIndexPath, + localSource.Signature, + delta.DeltaIdentityName, + delta.Version, + indexNameInPackage); } public WinGetLocalSource(string workingDirectory, Signature? signature) @@ -125,7 +254,18 @@ public string CreateIndex(string indexName, uint? majorVersion = null, uint? min WinGetFactory factory = new (); using IWinGetSQLiteIndex indexHelper = majorVersion == null ? factory.SQLiteIndexCreateLatestVersion(fullPath) : factory.SQLiteIndexCreate(fullPath, majorVersion.Value, minorVersion.GetValueOrDefault()); - Queue filesQueue = new(Directory.EnumerateFiles(this.workingDirectory, "*.yaml", SearchOption.AllDirectories)); + this.AddManifestsToIndex(indexHelper, Directory.EnumerateFiles(this.workingDirectory, "*.yaml", SearchOption.AllDirectories)); + + indexHelper.PrepareForPackaging(); + + return fullPath; + } + + // Adds every given manifest, retrying the ones that fail. A manifest can legitimately fail + // on the first attempt when it names a package dependency that is not in the index yet. + private void AddManifestsToIndex(IWinGetSQLiteIndex indexHelper, IEnumerable manifestFiles) + { + Queue filesQueue = new(manifestFiles); while (filesQueue.Count > 0) { int currentCount = filesQueue.Count; @@ -151,13 +291,16 @@ public string CreateIndex(string indexName, uint? majorVersion = null, uint? min throw new InvalidOperationException("Failed to add all manifests in directory to index."); } } - - indexHelper.PrepareForPackaging(); - - return fullPath; } - public string CreatePackage(string packageName, string inputAppxManifestFile, string indexPath, Signature? signature) + public string CreatePackage( + string packageName, + string inputAppxManifestFile, + string indexPath, + Signature? signature, + string? identityName = null, + string? identityVersion = null, + string? indexNameInPackage = null) { if (!File.Exists(inputAppxManifestFile)) { @@ -172,9 +315,9 @@ public string CreatePackage(string packageName, string inputAppxManifestFile, st string appxManifestFile = Path.Combine(this.workingDirectory, "AppxManifest.xml"); File.Copy(inputAppxManifestFile, appxManifestFile, true); - if (signature != null && signature.Publisher != null) + if ((signature != null && signature.Publisher != null) || identityName != null || identityVersion != null) { - Helpers.ModifyAppxManifestIdentity(appxManifestFile, signature.Publisher); + Helpers.ModifyAppxManifestIdentity(appxManifestFile, signature?.Publisher, identityName, identityVersion); } string mappingFile = Path.Combine(this.workingDirectory, "MappingFile.txt"); @@ -182,11 +325,18 @@ public string CreatePackage(string packageName, string inputAppxManifestFile, st { using StreamWriter outputFile = new(mappingFile, false); outputFile.WriteLine("[Files]"); - outputFile.WriteLine($"\"{indexPath}\" \"Public\\{Path.GetFileName(indexPath)}\""); + outputFile.WriteLine($"\"{indexPath}\" \"Public\\{indexNameInPackage ?? Path.GetFileName(indexPath)}\""); outputFile.WriteLine($"\"{appxManifestFile}\" \"AppxManifest.xml\""); } string outputPackage = Path.Combine(this.workingDirectory, packageName); + + string? outputDirectory = Path.GetDirectoryName(outputPackage); + if (!string.IsNullOrEmpty(outputDirectory)) + { + Directory.CreateDirectory(outputDirectory); + } + Helpers.PackWithMappingFile(outputPackage, mappingFile); if (signature != null)