Merge pull request #2637 from Azure/dev

Merge dev into main for v2.11.4 release

.github/ISSUE_TEMPLATE/new-release-template.md

@@ -14,6 +14,7 @@ _Due: <2-3-business-days-before-release>_
 - [ ] Delete DTFx test packages from the [ADO feed](https://dev.azure.com/durabletaskframework/Durable%20Task%20Framework%20CI/_artifacts/feed/durabletask).
 - [ ] Run the [DTFx release pipeline](https://durabletaskframework.visualstudio.com/Durable%20Task%20Framework%20CI/_build?definitionId=21) ([defined here](https://github.com/Azure/durabletask/blob/main/azure-pipelines-release.yml)) to obtain new packages.
 - [ ] Publish DTFx packages to the [ADO feed](https://dev.azure.com/durabletaskframework/Durable%20Task%20Framework%20CI/_artifacts/feed/durabletask) for testing.
+- [ ] Keep branch `azure-storage-v12` updated with branch `main`.
 
 **Prep Release (assigned to: )**
 _Due: <2-business-days-before-release>_
@@ -23,6 +24,7 @@ _Due: <2-business-days-before-release>_
 - [ ] Add the Durable Functions package to the [ADO test feed](https://dev.azure.com/durabletaskframework/Durable%20Task%20Framework%20CI/_artifacts/feed/durabletask-test).
 - [ ] Check for package size, make sure it's not surprisingly heavier than a previous release.
 - [ ] Merge (**choose create a merge commit, NOT squash merge**) dev into main. Person performing validation must approve PR.
+- [ ] Keep branch `v3.x` updated with branch `dev`. Do not merge PRs that are specific to Durable Functions v2.
 
 **Validation**
 _Due: <1-business-days-before-release>_

.github/pull_request_template.md

@@ -20,4 +20,6 @@ resolves #issue_for_this_pr
 * [ ] My changes **do not** change the version of the WebJobs.Extensions.DurableTask package
     * [ ] Otherwise: major or minor version updates are reflected in `/src/Worker.Extensions.DurableTask/AssemblyInfo.cs`
 * [ ] My changes **do not** add EventIds to our EventSource logs
-    * [ ] Otherwise: Ensure the EventIds are within the supported range in our existing Windows infrastructure. You may validate this with a deployed app's telemetry. You may also extend the range by completing a PR such as [this one](https://msazure.visualstudio.com/One/_git/AAPT-Antares-Websites/pullrequest/7463263?_a=files).
+    * [ ] Otherwise: Ensure the EventIds are within the supported range in our existing Windows infrastructure. You may validate this with a deployed app's telemetry. You may also extend the range by completing a PR such as [this one](https://msazure.visualstudio.com/One/_git/AAPT-Antares-Websites/pullrequest/7463263?_a=files).
+* [ ] My changes **should** be added to v3.x branch.
+    * [ ] Otherwise: This change only applies to Durable Functions v2.x and **will not** be merged to branch v3.x.

.github/workflows/smoketest-netherite-inproc-v4.yml

@@ -0,0 +1,22 @@
+name: Smoke Test - .NET in-proc w/ Netherite on Functions V4
+
+on:
+  push:
+    branches: [ main, dev ]
+    paths:
+      - 'src/**'
+      - 'test/SmokeTests/BackendSmokeTests/Netherite/**'
+  pull_request:
+    branches: [ main, dev ]
+    paths:
+      - 'src/**'
+      - 'test/SmokeTests/BackendSmokeTests/Netherite/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Run V4 .NET in-proc w/ Netherite Smoke Test
+      run: test/SmokeTests/e2e-test.ps1 -DockerfilePath test/SmokeTests/BackendSmokeTests/Netherite/Dockerfile -HttpStartPath api/DurableFunctionsHttpStart -ContainerName NetheriteApp
+      shell: pwsh

README.md

@@ -26,6 +26,7 @@ Package Name | NuGet
 ---|---
 Microsoft.Azure.WebJobs.Extensions.DurableTask | [![NuGet](https://img.shields.io/nuget/v/Microsoft.Azure.WebJobs.Extensions.DurableTask.svg)](https://www.nuget.org/packages/Microsoft.Azure.WebJobs.Extensions.DurableTask)
 Microsoft.Azure.WebJobs.Extensions.DurableTask.Analyzers (C# only) | [![NuGet](https://img.shields.io/nuget/v/Microsoft.Azure.WebJobs.Extensions.DurableTask.Analyzers.svg)](https://www.nuget.org/packages/Microsoft.Azure.WebJobs.Extensions.DurableTask.Analyzers)
+Microsoft.Azure.Functions.Worker.Extensions.DurableTask | [![NuGet](https://img.shields.io/nuget/v/Microsoft.Azure.Functions.Worker.Extensions.DurableTask.svg)](https://www.nuget.org/packages/Microsoft.Azure.Functions.Worker.Extensions.DurableTask)
 
 
 ## Language support

WebJobs.Extensions.DurableTask.sln

@@ -9,11 +9,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{7EC858EE-348
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{78BCF152-C22C-408F-9FB1-0F8C99B154B5}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{51D8DCEA-EA91-410E-AA6A-F42473F0C691}"
-	ProjectSection(SolutionItems) = preProject
-		.nuget\NuGet.Config = .nuget\NuGet.Config
-	EndProjectSection
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebJobs.Extensions.DurableTask.Tests.V2", "test\FunctionsV2\WebJobs.Extensions.DurableTask.Tests.V2.csproj", "{F2A5DABB-36D4-4152-AF49-2570149899E1}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebJobs.Extensions.DurableTask.Tests.V1", "test\FunctionsV1\WebJobs.Extensions.DurableTask.Tests.V1.csproj", "{F99CA746-553C-43B4-943B-59A5D190459A}"
@@ -23,6 +18,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		.editorconfig = .editorconfig
 		azure-pipelines-release-dotnet-isolated.yml = azure-pipelines-release-dotnet-isolated.yml
 		azure-pipelines-release.yml = azure-pipelines-release.yml
+		nuget.config = nuget.config
 		README.md = README.md
 		release_notes.md = release_notes.md
 		.stylecop\stylecop.json = .stylecop\stylecop.json

azure-pipelines-analyzer-release.yml

@@ -38,7 +38,7 @@ steps:
     command: restore
     projects: '**/**/*.csproj'
     feedsToUse: config
-    nugetConfigPath: '.nuget/nuget.config'
+    nugetConfigPath: 'nuget.config'
 
 # Build durable-analyzer
 - task: VSBuild@1

azure-pipelines-release.yml

@@ -38,7 +38,7 @@ steps:
     command: restore
     projects: '**/**/*.csproj'
     feedsToUse: config
-    nugetConfigPath: '.nuget/nuget.config'
+    nugetConfigPath: 'nuget.config'
 
 # Build durable-extension
 - task: VSBuild@1

azure-pipelines.yml

@@ -23,7 +23,7 @@ jobs:
         command: 'restore'
         projects: 'test/FunctionsV1/*.csproj'
         feedsToUse: 'config'
-        nugetConfigPath: '.nuget/nuget.config'
+        nugetConfigPath: 'nuget.config'
 
     - task: DotNetCoreCLI@2
       inputs:
@@ -72,7 +72,7 @@ jobs:
         command: 'restore'
         projects: 'test/FunctionsV2/*.csproj'
         feedsToUse: 'config'
-        nugetConfigPath: '.nuget/nuget.config'
+        nugetConfigPath: 'nuget.config'
 
     - task: DotNetCoreCLI@2
       inputs:
@@ -119,7 +119,7 @@ jobs:
         command: 'restore'
         projects: 'test/WebJobs.Extensions.DurableTask.Analyzers.Test/*.csproj'
         feedsToUse: 'config'
-        nugetConfigPath: '.nuget/nuget.config'
+        nugetConfigPath: 'nuget.config'
 
     - task: DotNetCoreCLI@2
       inputs:

nuget.config

@@ -3,8 +3,7 @@
 <configuration>
   <packageSources>
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
-    <add key="azure-appservice-staging" value="https://www.myget.org/F/azure-appservice-staging/api/v3/index.json" />
-    <add key="azure-appservice" value="https://www.myget.org/F/azure-appservice/api/v3/index.json" />
+    <add key="AzureFunctionsTempStaging" value="https://pkgs.dev.azure.com/azfunc/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsTempStaging/nuget/v3/index.json" />
     <add key="durabletask" value="https://pkgs.dev.azure.com/durabletaskframework/734e7913-2fab-4624-a174-bc57fe96f95d/_packaging/durabletask/nuget/v3/index.json" />
     <add key="durabletask-test" value="https://durabletaskframework.pkgs.visualstudio.com/734e7913-2fab-4624-a174-bc57fe96f95d/_packaging/durabletask-test/nuget/v3/index.json" />
   </packageSources>

release_notes.md

@@ -4,7 +4,7 @@
 
 ### Bug Fixes
 
-- Address input issues when using .NET isolated (#2581)[https://github.com/Azure/azure-functions-durable-extension/issues/2581]
+- Fix support for distributed tracing v2 in dotnet-isolated and Java (https://github.com/Azure/azure-functions-durable-extension/pull/2634)
 
 ### Breaking Changes
 

src/WebJobs.Extensions.DurableTask/AzureStorageDurabilityProvider.cs

@@ -10,6 +10,7 @@
 using DurableTask.AzureStorage.Tracking;
 using DurableTask.Core;
 using Microsoft.Extensions.Logging;
+using Microsoft.WindowsAzure.Storage;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Converters;
 using Newtonsoft.Json.Linq;
@@ -200,6 +201,16 @@ internal static OrchestrationInstanceStatusQueryCondition ConvertWebjobsDurableC
         }
 
 #if !FUNCTIONS_V1
+
+        internal DurableTaskMetricsProvider GetMetricsProvider(
+            string functionName,
+            string hubName,
+            CloudStorageAccount storageAccount,
+            ILogger logger)
+        {
+            return new DurableTaskMetricsProvider(functionName, hubName, logger, performanceMonitor: null, storageAccount);
+        }
+
         /// <inheritdoc/>
         public override bool TryGetScaleMonitor(
             string functionId,
@@ -208,12 +219,31 @@ public override bool TryGetScaleMonitor(
             string connectionName,
             out IScaleMonitor scaleMonitor)
         {
+            CloudStorageAccount storageAccount = this.storageAccountProvider.GetStorageAccountDetails(connectionName).ToCloudStorageAccount();
+            DurableTaskMetricsProvider metricsProvider = this.GetMetricsProvider(functionName, hubName, storageAccount, this.logger);
             scaleMonitor = new DurableTaskScaleMonitor(
                 functionId,
                 functionName,
                 hubName,
-                this.storageAccountProvider.GetStorageAccountDetails(connectionName).ToCloudStorageAccount(),
-                this.logger);
+                storageAccount,
+                this.logger,
+                metricsProvider);
+            return true;
+        }
+
+#endif
+#if FUNCTIONS_V3_OR_GREATER
+        public override bool TryGetTargetScaler(
+            string functionId,
+            string functionName,
+            string hubName,
+            string connectionName,
+            out ITargetScaler targetScaler)
+        {
+            // This is only called by the ScaleController, it doesn't run in the Functions Host process.
+            CloudStorageAccount storageAccount = this.storageAccountProvider.GetStorageAccountDetails(connectionName).ToCloudStorageAccount();
+            DurableTaskMetricsProvider metricsProvider = this.GetMetricsProvider(functionName, hubName, storageAccount, this.logger);
+            targetScaler = new DurableTaskTargetScaler(functionId, metricsProvider, this, this.logger);
             return true;
         }
 #endif

src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs

@@ -565,5 +565,27 @@ public virtual bool TryGetScaleMonitor(
             return false;
         }
 #endif
+
+#if FUNCTIONS_V3_OR_GREATER
+        /// <summary>
+        /// Tries to obtain a scaler for target based scaling.
+        /// </summary>
+        /// <param name="functionId">Function id.</param>
+        /// <param name="functionName">Function name.</param>
+        /// <param name="hubName">Task hub name.</param>
+        /// <param name="connectionName">The name of the storage-specific connection settings.</param>
+        /// <param name="targetScaler">The target-based scaler.</param>
+        /// <returns>True if target-based scaling is supported, false otherwise.</returns>
+        public virtual bool TryGetTargetScaler(
+            string functionId,
+            string functionName,
+            string hubName,
+            string connectionName,
+            out ITargetScaler targetScaler)
+        {
+            targetScaler = null;
+            return false;
+        }
+#endif
     }
 }

src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs

@@ -150,13 +150,13 @@ public DurableTaskExtension(
             this.nameResolver = nameResolver ?? throw new ArgumentNullException(nameof(nameResolver));
             this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
             this.PlatformInformationService = platformInformationService ?? throw new ArgumentNullException(nameof(platformInformationService));
-            this.ResolveAppSettingOptions();
+            DurableTaskOptions.ResolveAppSettingOptions(this.Options, this.nameResolver);
 
             ILogger logger = loggerFactory.CreateLogger(LoggerCategoryName);
 
             this.TraceHelper = new EndToEndTraceHelper(logger, this.Options.Tracing.TraceReplayEvents);
             this.LifeCycleNotificationHelper = lifeCycleNotificationHelper ?? this.CreateLifeCycleNotificationHelper();
-            this.durabilityProviderFactory = this.GetDurabilityProviderFactory(this.Options, logger, orchestrationServiceFactories);
+            this.durabilityProviderFactory = GetDurabilityProviderFactory(this.Options, logger, orchestrationServiceFactories);
             this.defaultDurabilityProvider = this.durabilityProviderFactory.GetDurabilityProvider();
             this.isOptionsConfigured = true;
 
@@ -249,6 +249,8 @@ public string HubName
 
         internal DurableTaskOptions Options { get; }
 
+        internal DurabilityProvider DefaultDurabilityProvider => this.defaultDurabilityProvider;
+
         internal HttpApiHandler HttpApiHandler { get; private set; }
 
         internal ILifeCycleNotificationHelper LifeCycleNotificationHelper { get; private set; }
@@ -296,7 +298,7 @@ private MessagePayloadDataConverter CreateErrorDataConverter(IErrorSerializerSet
             return new MessagePayloadDataConverter(errorSerializerSettingsFactory.CreateJsonSerializerSettings(), isDefault);
         }
 
-        private IDurabilityProviderFactory GetDurabilityProviderFactory(DurableTaskOptions options, ILogger logger, IEnumerable<IDurabilityProviderFactory> orchestrationServiceFactories)
+        internal static IDurabilityProviderFactory GetDurabilityProviderFactory(DurableTaskOptions options, ILogger logger, IEnumerable<IDurabilityProviderFactory> orchestrationServiceFactories)
         {
             bool storageTypeIsConfigured = options.StorageProvider.TryGetValue("type", out object storageType);
 
@@ -578,32 +580,13 @@ private void StopLocalGrpcServer()
         }
 #endif
 
-        private void ResolveAppSettingOptions()
-        {
-            if (this.Options == null)
-            {
-                throw new InvalidOperationException($"{nameof(this.Options)} must be set before resolving app settings.");
-            }
-
-            if (this.nameResolver == null)
-            {
-                throw new InvalidOperationException($"{nameof(this.nameResolver)} must be set before resolving app settings.");
-            }
-
-            if (this.nameResolver.TryResolveWholeString(this.Options.HubName, out string taskHubName))
-            {
-                // use the resolved task hub name
-                this.Options.HubName = taskHubName;
-            }
-        }
-
         private void InitializeForFunctionsV1(ExtensionConfigContext context)
         {
 #if FUNCTIONS_V1
             context.ApplyConfig(this.Options, "DurableTask");
             this.nameResolver = context.Config.NameResolver;
             this.loggerFactory = context.Config.LoggerFactory;
-            this.ResolveAppSettingOptions();
+            DurableTaskOptions.ResolveAppSettingOptions(this.Options, this.nameResolver);
             ILogger logger = this.loggerFactory.CreateLogger(LoggerCategoryName);
             this.TraceHelper = new EndToEndTraceHelper(logger, this.Options.Tracing.TraceReplayEvents);
             this.connectionInfoResolver = new WebJobsConnectionInfoProvider();
@@ -1573,59 +1556,6 @@ internal static void TagActivityWithOrchestrationStatus(OrchestrationRuntimeStat
                 activity.AddTag("DurableFunctionsRuntimeStatus", statusStr);
             }
         }
-
-        internal IScaleMonitor GetScaleMonitor(string functionId, FunctionName functionName, string connectionName)
-        {
-            if (this.defaultDurabilityProvider.TryGetScaleMonitor(
-                    functionId,
-                    functionName.Name,
-                    this.Options.HubName,
-                    connectionName,
-                    out IScaleMonitor scaleMonitor))
-            {
-                return scaleMonitor;
-            }
-            else
-            {
-                // the durability provider does not support runtime scaling.
-                // Create an empty scale monitor to avoid exceptions (unless runtime scaling is actually turned on).
-                return new NoOpScaleMonitor($"{functionId}-DurableTaskTrigger-{this.Options.HubName}".ToLower());
-            }
-        }
-
-        /// <summary>
-        /// A placeholder scale monitor, can be used by durability providers that do not support runtime scaling.
-        /// This is required to allow operation of those providers even if runtime scaling is turned off
-        /// see discussion https://github.com/Azure/azure-functions-durable-extension/pull/1009/files#r341767018.
-        /// </summary>
-        private sealed class NoOpScaleMonitor : IScaleMonitor
-        {
-            /// <summary>
-            /// Construct a placeholder scale monitor.
-            /// </summary>
-            /// <param name="name">A descriptive name.</param>
-            public NoOpScaleMonitor(string name)
-            {
-                this.Descriptor = new ScaleMonitorDescriptor(name);
-            }
-
-            /// <summary>
-            /// A descriptive name.
-            /// </summary>
-            public ScaleMonitorDescriptor Descriptor { get; private set; }
-
-            /// <inheritdoc/>
-            Task<ScaleMetrics> IScaleMonitor.GetMetricsAsync()
-            {
-                throw new InvalidOperationException("The current DurableTask backend configuration does not support runtime scaling");
-            }
-
-            /// <inheritdoc/>
-            ScaleStatus IScaleMonitor.GetScaleStatus(ScaleStatusContext context)
-            {
-                throw new InvalidOperationException("The current DurableTask backend configuration does not support runtime scaling");
-            }
-        }
 #endif
     }
 }