Update build pipeline to support LocalPackage build config (parameter…

…ized; not enabled by default) (#20552)

BuildConfigGen/GitUtil.cs

@@ -95,30 +95,6 @@ public static void GetUntrackedFiles(string taskTarget, out IEnumerable<string>
             toAdd = untrackedOuput2;
             toRemove = untrackedOuputRemove;
         }
-        internal static string GetGitRootPath(string currentDir)
-        {
-            const string args = "rev-parse --git-dir";
-            string path = RunGitCommandScalar(currentDir, args);
-
-            path = FixupPath(path);
-
-            const string gitDir = ".git";
-            if (path.EndsWith(gitDir))
-            {
-                path = path.Substring(0, path.Length - gitDir.Length);
-
-                if (path == "")
-                {
-                    return currentDir;
-                }
-
-                return path;
-            }
-            else
-            {
-                throw new Exception($"expected git {args} to return  ");
-            }
-        }
 
         internal static IEnumerable<string> GetNonIgnoredFileListFromPath(string gitRoot, string taskTarget)
         {

BuildConfigGen/Program.cs

@@ -109,7 +109,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
             }
 
             string currentDir = Environment.CurrentDirectory;
-            string gitRootPath = GitUtil.GetGitRootPath(currentDir);
+            string gitRootPath = GetTasksRootPath(currentDir);
 
             string globalVersionPath = Path.Combine(gitRootPath, @"globalversion.txt");
             TaskVersion? globalVersion = GetGlobalVersion(gitRootPath, globalVersionPath);
@@ -208,7 +208,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
                         Console.WriteLine($"Global version: maxPatchForCurrentSprint = maxPatchForCurrentSprint + 1");
                     }
 
-                    Console.WriteLine($"Global version update: globalVersion = {globalVersion} maxPatchForCurrentSprint={maxPatchForCurrentSprint}" );
+                    Console.WriteLine($"Global version update: globalVersion = {globalVersion} maxPatchForCurrentSprint={maxPatchForCurrentSprint}");
                 }
                 else
                 {
@@ -249,6 +249,14 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
                             }
                         }
                     }
+                    else
+                    {
+                        // if we're not updating local packages, we need to ensure the global version is updated to the task major version. existing patch number is preserved
+                        if (globalVersion is not null)
+                        {
+                            globalVersion = globalVersion.CloneWithMajor(taskMajorVersion);
+                        }
+                    }
 
                     if (globalVersion is not null)
                     {
@@ -265,7 +273,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
                     ensureUpdateModeVerifier!.WriteAllText(globalVersionPath, globalVersion!.MinorPatchToString(), false);
                 }
 
-                ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError("(global)", skipContentCheck: false);
+                ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(null, skipContentCheck: false);
 
                 foreach (var t in tasks)
                 {
@@ -284,6 +292,37 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
             }
         }
 
+        private static string GetTasksRootPath(string inputCurrentDir)
+        {
+            string? currentDir = inputCurrentDir;
+            string? tasksRootPath = null;
+
+            do
+            {
+                string currentDirGit = Path.Combine(currentDir, ".git");
+
+                if (Directory.Exists(currentDirGit))
+                {
+                    tasksRootPath = currentDir;
+                }
+
+                currentDir = (new DirectoryInfo(currentDir)).Parent?.FullName;
+
+            } while (currentDir != null);
+
+            if (tasksRootPath == null)
+            {
+                throw new Exception($"could not find .git in {currentDir}");
+            }
+
+            if (!File.Exists(Path.Combine(tasksRootPath, "make-options.json")))
+            {
+                throw new Exception($"make-options.json not found in tasksRootPath={tasksRootPath}");
+            }
+
+            return tasksRootPath;
+        }
+
         private static IEnumerable<string> FilterConfigsForTask(string? configs, KeyValuePair<string, MakeOptionsReader.AgentTask> t)
         {
             var configsList = t.Value.Configs.AsEnumerable();
@@ -343,7 +382,7 @@ private static void GetVersions(string task, string configsString, out List<(str
 
             string currentDir = Environment.CurrentDirectory;
 
-            string gitRootPath = GitUtil.GetGitRootPath(currentDir);
+            string gitRootPath = GetTasksRootPath(currentDir);
 
             string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task);
             if (!Directory.Exists(taskTargetPath))
@@ -419,7 +458,7 @@ private static int GetCurrentSprint()
             return currentSprint;
         }
 
-        private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(string task, bool skipContentCheck)
+        private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(string? task, bool skipContentCheck)
         {
             // if !writeUpdates, error if we have written any updates
             var verifyErrors = ensureUpdateModeVerifier!.GetVerifyErrors(skipContentCheck).ToList();
@@ -433,7 +472,14 @@ private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferEr
                     Console.WriteLine(s);
                 }
 
-                throw new Exception($"Updates needed, please run npm make.js --task {task} ");
+                if (task is null)
+                {
+                    throw new Exception($"Updates needed, please run node make.js");
+                }
+                else
+                {
+                    throw new Exception($"Updates needed, please run node make.js --task {task} ");
+                }
             }
         }
 
@@ -459,8 +505,8 @@ private static void MainUpdateTask(
             try
             {
                 string currentDir = Environment.CurrentDirectory;
-                string gitRootPath = GitUtil.GetGitRootPath(currentDir);
-                string versionMapFile = GetVersionMapFile(task, gitRootPath, generatedFolder);
+                string gitRootPath = GetTasksRootPath(currentDir);
+                string versionMapFile = GetVersionMapFile(task, generatedFolder);
 
                 string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task);
                 if (!Directory.Exists(taskTargetPath))
@@ -489,7 +535,11 @@ private static void MainUpdateTask(
 
                 foreach (var config in targetConfigs)
                 {
-                    if (config.useGlobalVersion && !hasGlobalVersion)
+                    if (config.useGlobalVersion && !includeLocalPackagesBuildConfig)
+                    {
+                        Console.WriteLine($"Info: MainUpdateTask: Skipping useGlobalVersion config for task b/c --include-local-packages-build-config. not specified. hasGlobalVersion={hasGlobalVersion} config.useGlobalVersion={config.useGlobalVersion} includeLocalPackagesBuildConfig={includeLocalPackagesBuildConfig}");
+                    }
+                    else if (config.useGlobalVersion && !hasGlobalVersion)
                     {
                         Console.WriteLine($"Info: MainUpdateTask: Skipping useGlobalVersion config for task b/c GlobalVersion not initialized.  (to opt-in and start producing LocalBuildConfig, run with --include-local-packages-build-config.  hasGlobalVersion={hasGlobalVersion} config.useGlobalVersion={config.useGlobalVersion}).  Note: this is not an error!");
                     }
@@ -608,7 +658,7 @@ private static void MainUpdateTask(
             }
         }
 
-        private static string GetVersionMapFile(string task, string gitRootPath, string generatedFolder)
+        private static string GetVersionMapFile(string task, string generatedFolder)
         {
             return Path.Combine(generatedFolder, @$"{task}.versionmap.txt");
         }
@@ -1078,7 +1128,7 @@ private static void CopyConfig(string gitRootPath, string taskTargetPathOrUnders
         private static void UpdateVersionsForTask(string task, TaskStateStruct taskState, HashSet<Config.ConfigRecord> targetConfigs, int currentSprint, string globalVersionPath, TaskVersion? globalVersion, string generatedFolder)
         {
             string currentDir = Environment.CurrentDirectory;
-            string gitRootPath = GitUtil.GetGitRootPath(currentDir);
+            string gitRootPath = GetTasksRootPath(currentDir);
             string taskTargetPath = Path.Combine(gitRootPath, "Tasks", task);
 
             if (!Directory.Exists(taskTargetPath))
@@ -1093,7 +1143,7 @@ private static void UpdateVersionsForTask(string task, TaskStateStruct taskState
 
             bool defaultVersionMatchesSourceVersion;
 
-            string versionMapFile = GetVersionMapFile(task, gitRootPath, generatedFolder);
+            string versionMapFile = GetVersionMapFile(task, generatedFolder);
 
             {
                 TaskVersion? defaultVersion = null;

BuildConfigGen/dev.sh

@@ -36,17 +36,6 @@ function detect_platform_and_runtime_id ()
     fi
 }
 
-function cmd_build ()
-{
-    heading "Building"
-    dotnet build -o bin $SOLUTION_PATH || failed build
-    #change execution flag to allow running with sudo
-    if [[ ("$CURRENT_PLATFORM" == "linux") || ("$CURRENT_PLATFORM" == "darwin") ]]; then
-        chmod +x "bin/BuildConfigGen"
-    fi
-
-}
-
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 pushd "$SCRIPT_DIR"
 source "$SCRIPT_DIR/Misc/helpers.sh"
@@ -91,6 +80,3 @@ echo "Adding .NET to PATH ${DOTNETSDK_INSTALLDIR}"
 export PATH=${DOTNETSDK_INSTALLDIR}:$PATH
 echo "Path = $PATH"
 echo ".NET Version = $(dotnet --version)"
-
-SOLUTION_PATH="$SCRIPT_DIR/BuildConfigGen.sln"
-cmd_build

azure-pipelines.yml

@@ -29,6 +29,10 @@ parameters:
   displayName: Enable CodeQL for run
   type: boolean
   default: false
+- name: includeLocalPackagesBuildConfig
+  displayName: Flag to update LocalPackages buildconfig (for testing, this will be made  default later)
+  type: boolean
+  default: false # note: keep in sync with ci\ci-test-tasks\canary-tests-v2.yml
 
 variables:
 - name: currentDate
@@ -41,7 +45,16 @@ variables:
   value: ${{ eq(parameters.enableCodeQL, true) }}
 - name: system.debug
   value: true
-
+- name: includeLocalPackagesBuildConfigParameter
+  ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}:
+    value: '--includeLocalPackagesBuildConfig'
+  ${{ else }}:
+    value: ''
+- name: IncludeLocalPackagesBuildConfigTest
+  ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}:
+    value: '1'
+  ${{ else }}:
+    value: ''
 
 extends:
   template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates

ci/build-all-steps.yml

@@ -106,7 +106,7 @@ steps:
   displayName: Clean tasks
 
 # Build Tasks
-- script: node make.js serverBuild --task "$(getTaskPattern.task_pattern)"
+- script: node make.js serverBuild --task "$(getTaskPattern.task_pattern)" $(includeLocalPackagesBuildConfigParameter)
   displayName: Build Tasks
   condition: and(succeeded(), ne(variables['numTasks'], 0))
 

ci/build-all-tasks.yml

@@ -30,12 +30,12 @@ steps:
   displayName: Clean tasks
 
 # Build tasks
-- script: node make.js serverBuild --task "$(task_pattern)"
+- script: node make.js serverBuild --task "$(task_pattern)" $(includeLocalPackagesBuildConfigParameter)
   displayName: Build tasks using pattern
   condition: and(succeeded(), eq('${{ parameters.deploy_all_tasks }}', false))
 
 # Build all tasks
-- script: node make.js serverBuild
+- script: node make.js serverBuild $(includeLocalPackagesBuildConfigParameter)
   displayName: Build all tasks
   condition: and(succeeded(), eq('${{ parameters.deploy_all_tasks }}', true))
 

ci/build-single-steps.yml

@@ -50,7 +50,7 @@ steps:
   displayName: Clean tasks
 
 # Build Tasks
-- script: node make.js serverBuild --task "$(task_pattern)"
+- script: node make.js serverBuild --task "$(task_pattern)" $(includeLocalPackagesBuildConfigParameter)
   displayName: Build Tasks
 
 # Check diff for task sources

ci/ci-test-tasks/build-task.yml

@@ -22,7 +22,7 @@ steps:
   displayName: Clean tasks
 
 # Build
-- script: node make.js serverBuild --task "${{ parameters.task }}"
+- script: node make.js serverBuild --task "${{ parameters.task }}" $(includeLocalPackagesBuildConfigParameter)
   displayName: ${{ parameters.task }} build
 
 # Check diff for task sources

ci/ci-test-tasks/canary-tests-v2.yml

@@ -1,3 +1,21 @@
+parameters:
+- name: includeLocalPackagesBuildConfig
+  displayName: Flag to update LocalPackages buildconfig (for testing, this will be made  default later)
+  type: boolean
+  default: false # Note: keep in sync with /azure-pipelines.yml
+
+variables:
+- name: includeLocalPackagesBuildConfigParameter
+  ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}: 
+    value: '--includeLocalPackagesBuildConfig'
+  ${{ else }}:
+    value: ''
+- name: IncludeLocalPackagesBuildConfigTest
+  ${{ if eq(parameters.includeLocalPackagesBuildConfig, true) }}:
+    value: '1'
+  ${{ else }}:
+    value: ''
+
 trigger:
   - master
   - releases/*

make-util.js

@@ -1747,28 +1747,22 @@ exports.renameCodeCoverageOutput = renameCodeCoverageOutput;
 //------------------------------------------------------------------------------
 
 /**
- * Returns path to BuldConfigGenerator, build it if needed.  Fail on compilation failure
+ * Ensure Pre-reqs for buildConfigGen (e.g. dotnet)
  * @param {String} baseConfigToolPath base build config tool path
- * @returns {String} Path to the executed file
  */
-var getBuildConfigGenerator = function (baseConfigToolPath) {
-    var programPath = "";
+var ensureBuildConfigGeneratorPrereqs = function (baseConfigToolPath) {
     var configToolBuildUtility = "";
 
     if (os.platform() === 'win32') {
-        programPath = path.join(baseConfigToolPath, 'bin', 'BuildConfigGen.exe');
         configToolBuildUtility = path.join(baseConfigToolPath, "dev.cmd");
     } else {
-        programPath = path.join(baseConfigToolPath, 'bin', 'BuildConfigGen');
         configToolBuildUtility = path.join(baseConfigToolPath, "dev.sh");
     }
 
     // build configToolBuildUtility if needed.  (up-to-date check will skip build if not needed)
     run(configToolBuildUtility, true);
-
-    return programPath;
 };
-exports.getBuildConfigGenerator = getBuildConfigGenerator;
+exports.ensureBuildConfigGeneratorPrereqs = ensureBuildConfigGeneratorPrereqs;
 
 /**
  * Function to validate or write generated tasks
@@ -1785,7 +1779,9 @@ var processGeneratedTasks = function(baseConfigToolPath, taskList, makeOptions,
     if (sprintNumber && !Number.isInteger(sprintNumber)) fail("Sprint is not a number");
 
     var tasks = taskList.join('|')
-    const programPath = getBuildConfigGenerator(baseConfigToolPath);
+    ensureBuildConfigGeneratorPrereqs(baseConfigToolPath);
+    var programPath = `dotnet run --project "${baseConfigToolPath}/BuildConfigGen.csproj" -- `
+
     const args = [
         "--task",
         `"${tasks}"`