Merge branch 'main' into copilot/fix-1286

Author: dylan-smith

.github/copilot-instructions.md

@@ -0,0 +1,27 @@
+This is a C# based repository that produces several CLIs that are used by customers to interact with the GitHub migration APIs. Please follow these guidelines when contributing:
+
+## Code Standards
+
+### Required Before Each Commit
+- Run `dotnet format src/OctoshiftCLI.sln` before committing any changes to ensure proper code formatting. This will run dotnet format on all C# files to maintain consistent style
+
+### Development Flow
+- Build: `dotnet build src/OctoshiftCLI.sln /p:TreatWarningsAsErrors=true`
+- Test: `dotnet test src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj`
+
+## Repository Structure
+- `src/`: Contains the main C# source code for the Octoshift CLI
+- `src/ado2gh/`: Contains the ADO to GH CLI commands
+- `src/bbs2gh/`: Contains the BBS to GH CLI commands
+- `src/gei/`: Contains the GitHub to GitHub CLI commands
+- `src/Octoshift/`: Contains shared logic used by multiple commands/CLIs
+- `src/OctoshiftCLI.IntegrationTests/`: Contains integration tests for the Octoshift CLI
+- `src/OctoshiftCLI.Tests/`: Contains unit tests for the Octoshift CLI
+
+## Key Guidelines
+1. Follow C# best practices and idiomatic patterns
+2. Maintain existing code structure and organization
+4. Write unit tests for new functionality.
+5. When making changes that would impact our users (e.g. new features or bug fixes), add a bullet point to `RELEASENOTES.md` with a user friendly brief description of the change
+6. Never silently swallow exceptions.
+7. If an exception is expected/understood and we can give a helpful user-friendly message, then throw an OctoshiftCliException with a user-friendly message. Otherwise let the exception bubble up and the top-level exception handler will log and handle it appropriately.

.github/workflows/copilot-setup-steps.yml

@@ -0,0 +1,29 @@
+name: Setup Development Environment
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - .github/workflows/copilot-setup-steps.yml
+  pull_request:
+    paths:
+      - .github/workflows/copilot-setup-steps.yml
+
+permissions:
+  contents: read
+
+jobs:
+  # The job name MUST be 'copilot-setup-steps' to be picked up by GitHub Copilot
+  copilot-setup-steps:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v2
+        with:
+          global-json-file: global.json
+
+      - name: Restore dependencies
+        run: dotnet restore src/OctoshiftCLI.sln

.github/workflows/integration-tests.yml

@@ -6,6 +6,14 @@ on:
       pr_number:
         type: number
         required: true
+      sha:
+        type: string
+        required: true
+
+permissions:
+  contents: read
+  checks: write
+  pull-requests: write
 
 jobs:
   build-for-e2e-test:
@@ -20,6 +28,19 @@ jobs:
         ref: 'refs/pull/${{ github.event.inputs.pr_number }}/merge'
         fetch-depth: 0
 
+    - name: Check SHA
+      run: |
+        git fetch origin refs/pull/${{ github.event.inputs.pr_number }}/head:pr-head
+        prsha=`git rev-parse pr-head | awk '{ print $1 }'`
+
+        echo "PR SHA: $prsha"
+        echo "expected SHA: ${{ github.event.inputs.SHA }}"
+
+        if [ "$prsha" != "${{ github.event.inputs.SHA }}" ]; then
+          echo "SHA must match" >&2
+          exit 1
+        fi
+
     - name: Setup .NET
       uses: actions/setup-dotnet@v2
       with:
@@ -67,6 +88,7 @@ jobs:
   e2e-test:
     needs: [ build-for-e2e-test ]
     strategy:
+      fail-fast: false
       matrix:
         runner-os: [windows-latest, ubuntu-latest, macos-latest]
         source-vcs: [AdoBasic, AdoCsv, Bbs, Ghes, Github]

LATEST-VERSION.txt

@@ -1 +1 @@
-v1.15.1
+v1.16.0

RELEASENOTES.md

@@ -1 +1 @@
-
+- Fixed `gh gei migrate-secret-alerts` command to handle long resolution comments by truncating them to fit within GitHub's 270 character limit while preserving the resolver name prefix.

releasenotes/v1.16.0.md

@@ -0,0 +1,3 @@
+- Fixed `ado2gh integrate-boards` command to properly report errors when GitHub PAT permissions are incorrect, instead of incorrectly reporting success.
+- Added `--target-api-url` option to `gh ado2gh rewire-pipeline` command to support customers migrating to [GitHub Enterprise Cloud with data residency](https://docs.github.com/en/enterprise-cloud@latest/admin/data-residency/about-github-enterprise-cloud-with-data-residency)
+- Updated `ado2gh generate-script` to no longer include ADO Boards integration commands in the generated script. This is a temporary change to fix broken functionality until we can ship a longer term fix. More details can be found [here](https://github.com/github/gh-gei/issues/1357)

src/Octoshift/Services/AdoApi.cs

@@ -212,16 +212,19 @@ public virtual async Task<string> GetGithubHandle(string org, string teamProject
         };
 
         var response = await _client.PostAsync(url, payload);
-        var data = JObject.Parse(response);
 
-#pragma warning disable IDE0046 // Convert to conditional expression
-        if (data["dataProviders"]["ms.vss-work-web.github-user-data-provider"] == null)
+        // Check for error message in the response
+        var errorMessage = ExtractErrorMessage(response, "ms.vss-work-web.github-user-data-provider");
+        if (errorMessage.HasValue())
         {
-            throw new OctoshiftCliException("Missing data from 'ms.vss-work-web.github-user-data-provider'. Please ensure the Azure DevOps project has a configured GitHub connection.");
+            throw new OctoshiftCliException($"Error validating GitHub token: {errorMessage}");
         }
-#pragma warning restore IDE0046 // Convert to conditional expression
 
-        return (string)data["dataProviders"]["ms.vss-work-web.github-user-data-provider"]["login"];
+        var data = JObject.Parse(response);
+        var dataProviders = data["dataProviders"] ?? throw new OctoshiftCliException("Missing data from 'ms.vss-work-web.github-user-data-provider'. Please ensure the Azure DevOps project has a configured GitHub connection.");
+        var dataProvider = dataProviders["ms.vss-work-web.github-user-data-provider"] ?? throw new OctoshiftCliException("Missing data from 'ms.vss-work-web.github-user-data-provider'. Please ensure the Azure DevOps project has a configured GitHub connection.");
+
+        return (string)dataProvider["login"];
     }
 
     public virtual async Task<(string connectionId, string endpointId, string connectionName, IEnumerable<string> repoIds)> GetBoardsGithubConnection(string org, string teamProject)
@@ -329,7 +332,14 @@ public virtual async Task AddRepoToBoardsGithubConnection(string org, string tea
             }
         };
 
-        await _client.PostAsync(url, payload);
+        var response = await _client.PostAsync(url, payload);
+
+        // Check for error message in the response
+        var errorMessage = ExtractErrorMessage(response, "ms.vss-work-web.azure-boards-save-external-connection-data-provider");
+        if (errorMessage.HasValue())
+        {
+            throw new OctoshiftCliException($"Error adding repository to boards GitHub connection: {errorMessage}");
+        }
     }
 
     public virtual async Task<string> GetTeamProjectId(string org, string teamProject)
@@ -523,34 +533,59 @@ public virtual async Task ShareServiceConnection(string adoOrg, string adoTeamPr
         return (defaultBranch, clean, checkoutSubmodules);
     }
 
-    public virtual async Task ChangePipelineRepo(string adoOrg, string teamProject, int pipelineId, string defaultBranch, string clean, string checkoutSubmodules, string githubOrg, string githubRepo, string connectedServiceId)
+    public virtual async Task ChangePipelineRepo(string adoOrg, string teamProject, int pipelineId, string defaultBranch, string clean, string checkoutSubmodules, string githubOrg, string githubRepo, string connectedServiceId, string targetApiUrl = null)
     {
         var url = $"{_adoBaseUrl}/{adoOrg.EscapeDataString()}/{teamProject.EscapeDataString()}/_apis/build/definitions/{pipelineId}?api-version=6.0";
 
         var response = await _client.GetAsync(url);
         var data = JObject.Parse(response);
 
+        // Determine base URLs
+        string apiUrl, webUrl, cloneUrl, branchesUrl, refsUrl, manageUrl;
+        if (targetApiUrl.HasValue())
+        {
+            var apiUri = new Uri(targetApiUrl.TrimEnd('/'));
+            var webHost = apiUri.Host.StartsWith("api.") ? apiUri.Host[4..] : apiUri.Host;
+            var webScheme = apiUri.Scheme;
+            var webBase = $"{webScheme}://{webHost}";
+            apiUrl = $"{targetApiUrl.TrimEnd('/')}/repos/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}";
+            webUrl = $"{webBase}/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}";
+            cloneUrl = $"{webBase}/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}.git";
+            branchesUrl = $"{targetApiUrl.TrimEnd('/')}/repos/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}/branches";
+            refsUrl = $"{targetApiUrl.TrimEnd('/')}/repos/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}/git/refs";
+            manageUrl = webUrl;
+        }
+        else
+        {
+            apiUrl = $"https://api.github.com/repos/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}";
+            webUrl = $"https://github.com/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}";
+            cloneUrl = $"https://github.com/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}.git";
+            branchesUrl = $"https://api.github.com/repos/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}/branches";
+            refsUrl = $"https://api.github.com/repos/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}/git/refs";
+            manageUrl = webUrl;
+        }
+
         var newRepo = new
         {
             properties = new
             {
-                apiUrl = $"https://api.github.com/repos/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}",
-                branchesUrl = $"https://api.github.com/repos/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}/branches",
-                cloneUrl = $"https://github.com/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}.git",
+                apiUrl,
+                branchesUrl,
+                cloneUrl,
                 connectedServiceId,
                 defaultBranch,
                 fullName = $"{githubOrg}/{githubRepo}",
-                manageUrl = $"https://github.com/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}",
+                manageUrl,
                 orgName = githubOrg,
-                refsUrl = $"https://api.github.com/repos/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}/git/refs",
+                refsUrl,
                 safeRepository = $"{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}",
                 shortName = githubRepo,
                 reportBuildStatus = true
             },
             id = $"{githubOrg}/{githubRepo}",
             type = "GitHub",
             name = $"{githubOrg}/{githubRepo}",
-            url = $"https://github.com/{githubOrg.EscapeDataString()}/{githubRepo.EscapeDataString()}.git",
+            url = cloneUrl,
             defaultBranch,
             clean,
             checkoutSubmodules
@@ -600,9 +635,26 @@ public virtual async Task<string> GetBoardsGithubRepoId(string org, string teamP
         };
 
         var response = await _client.PostAsync(url, payload);
+
+        // Check for error message in the response
+        var errorMessage = ExtractErrorMessage(response, "ms.vss-work-web.github-user-repository-data-provider");
+        if (errorMessage.HasValue())
+        {
+            throw new OctoshiftCliException($"Error getting GitHub repository information: {errorMessage}");
+        }
+
         var data = JObject.Parse(response);
+        var dataProviders = data["dataProviders"] ?? throw new OctoshiftCliException("Could not retrieve GitHub repository information. Please verify the repository exists and the GitHub token has the correct permissions.");
+        var dataProvider = dataProviders["ms.vss-work-web.github-user-repository-data-provider"];
 
-        return (string)data["dataProviders"]["ms.vss-work-web.github-user-repository-data-provider"]["additionalProperties"]["nodeId"];
+#pragma warning disable IDE0046 // Convert to conditional expression
+        if (dataProvider == null || dataProvider["additionalProperties"] == null || dataProvider["additionalProperties"]["nodeId"] == null)
+#pragma warning restore IDE0046 // Convert to conditional expression
+        {
+            throw new OctoshiftCliException("Could not retrieve GitHub repository information. Please verify the repository exists and the GitHub token has the correct permissions.");
+        }
+
+        return (string)dataProvider["additionalProperties"]["nodeId"];
     }
 
     public virtual async Task CreateBoardsGithubConnection(string org, string teamProject, string endpointId, string repoId)
@@ -641,7 +693,14 @@ public virtual async Task CreateBoardsGithubConnection(string org, string teamPr
             }
         };
 
-        await _client.PostAsync(url, payload);
+        var response = await _client.PostAsync(url, payload);
+
+        // Check for error message in the response
+        var errorMessage = ExtractErrorMessage(response, "ms.vss-work-web.azure-boards-save-external-connection-data-provider");
+        if (errorMessage.HasValue())
+        {
+            throw new OctoshiftCliException($"Error creating boards GitHub connection: {errorMessage}");
+        }
     }
 
     public virtual async Task DisableRepo(string org, string teamProject, string repoId)
@@ -700,6 +759,24 @@ public virtual async Task<bool> IsCallerOrgAdmin(string org)
         return await HasPermission(org, collectionSecurityNamespaceId, genericWritePermissionBitMaskValue);
     }
 
+    private string ExtractErrorMessage(string response, string dataProviderKey)
+    {
+        if (!response.HasValue())
+        {
+            return null;
+        }
+
+        var data = JObject.Parse(response);
+#pragma warning disable IDE0046 // Convert to conditional expression
+        if (data["dataProviders"] is not JObject dataProviders)
+        {
+            return null;
+        }
+#pragma warning restore IDE0046 // Convert to conditional expression
+
+        return dataProviders[dataProviderKey] is not JObject dataProvider ? null : (string)dataProvider["errorMessage"];
+    }
+
     private async Task<bool> HasPermission(string org, string securityNamespaceId, int permission)
     {
         var response = await _client.GetAsync($"{_adoBaseUrl}/{org.EscapeDataString()}/_apis/permissions/{securityNamespaceId.EscapeDataString()}/{permission}?api-version=6.0");

src/Octoshift/Services/SecretScanningAlertService.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
@@ -74,7 +75,13 @@ public virtual async Task MigrateSecretScanningAlerts(string sourceOrg, string s
 
                         _log.LogInformation($"  updating target alert:{targetAlert.Alert.Number} to state:{sourceAlert.Alert.State} and resolution:{sourceAlert.Alert.Resolution}");
 
-                        var targetResolutionComment = $"[@{sourceAlert.Alert.ResolverName}] {sourceAlert.Alert.ResolutionComment}";
+                        var prefix = $"[@{sourceAlert.Alert.ResolverName}] ";
+                        var originalComment = sourceAlert.Alert.ResolutionComment ?? string.Empty;
+                        var prefixedComment = prefix + originalComment;
+
+                        var targetResolutionComment = prefixedComment.Length <= 270
+                            ? prefixedComment
+                            : prefix + originalComment[..Math.Max(0, 270 - prefix.Length)];
 
                         await _targetGithubApi.UpdateSecretScanningAlert(targetOrg, targetRepo, targetAlert.Alert.Number, sourceAlert.Alert.State,
                             sourceAlert.Alert.Resolution, targetResolutionComment);

src/OctoshiftCLI.IntegrationTests/AdoBasicToGithub.cs

@@ -47,8 +47,6 @@ await retryPolicy.Retry(async () =>
             await Helper.AssertGithubRepoExists(githubOrg, $"{teamProject2}-{teamProject2}");
             await Helper.AssertGithubRepoInitialized(githubOrg, $"{teamProject1}-{teamProject1}");
             await Helper.AssertGithubRepoInitialized(githubOrg, $"{teamProject2}-{teamProject2}");
-            await Helper.AssertAutolinkConfigured(githubOrg, $"{teamProject1}-{teamProject1}", $"https://dev.azure.com/{adoOrg}/{teamProject1}/_workitems/edit/<num>/");
-            await Helper.AssertAutolinkConfigured(githubOrg, $"{teamProject2}-{teamProject2}", $"https://dev.azure.com/{adoOrg}/{teamProject2}/_workitems/edit/<num>/");
             await Helper.AssertAdoRepoDisabled(adoOrg, teamProject1, adoRepo1);
             await Helper.AssertAdoRepoDisabled(adoOrg, teamProject2, adoRepo2);
             await Helper.AssertAdoRepoLocked(adoOrg, teamProject1, adoRepo1);
@@ -69,8 +67,6 @@ await retryPolicy.Retry(async () =>
             await Helper.AssertServiceConnectionWasShared(adoOrg, teamProject2);
             await Helper.AssertPipelineRewired(adoOrg, teamProject1, pipeline1, githubOrg, $"{teamProject1}-{teamProject1}");
             await Helper.AssertPipelineRewired(adoOrg, teamProject2, pipeline2, githubOrg, $"{teamProject2}-{teamProject2}");
-            await Helper.AssertBoardsIntegrationConfigured(adoOrg, teamProject1);
-            await Helper.AssertBoardsIntegrationConfigured(adoOrg, teamProject2);
             Helper.AssertMigrationLogFileExists(githubOrg, $"{teamProject1}-{teamProject1}");
             Helper.AssertMigrationLogFileExists(githubOrg, $"{teamProject2}-{teamProject2}");
         }

src/OctoshiftCLI.IntegrationTests/AdoCsvToGithub.cs

@@ -48,8 +48,6 @@ await retryPolicy.Retry(async () =>
             await Helper.AssertGithubRepoExists(githubOrg, $"{teamProject2}-{teamProject2}");
             await Helper.AssertGithubRepoInitialized(githubOrg, $"{teamProject1}-{teamProject1}");
             await Helper.AssertGithubRepoInitialized(githubOrg, $"{teamProject2}-{teamProject2}");
-            await Helper.AssertAutolinkConfigured(githubOrg, $"{teamProject1}-{teamProject1}", $"https://dev.azure.com/{adoOrg}/{teamProject1}/_workitems/edit/<num>/");
-            await Helper.AssertAutolinkConfigured(githubOrg, $"{teamProject2}-{teamProject2}", $"https://dev.azure.com/{adoOrg}/{teamProject2}/_workitems/edit/<num>/");
             await Helper.AssertAdoRepoDisabled(adoOrg, teamProject1, adoRepo1);
             await Helper.AssertAdoRepoDisabled(adoOrg, teamProject2, adoRepo2);
             await Helper.AssertAdoRepoLocked(adoOrg, teamProject1, adoRepo1);
@@ -70,8 +68,6 @@ await retryPolicy.Retry(async () =>
             await Helper.AssertServiceConnectionWasShared(adoOrg, teamProject2);
             await Helper.AssertPipelineRewired(adoOrg, teamProject1, pipeline1, githubOrg, $"{teamProject1}-{teamProject1}");
             await Helper.AssertPipelineRewired(adoOrg, teamProject2, pipeline2, githubOrg, $"{teamProject2}-{teamProject2}");
-            await Helper.AssertBoardsIntegrationConfigured(adoOrg, teamProject1);
-            await Helper.AssertBoardsIntegrationConfigured(adoOrg, teamProject2);
             Helper.AssertMigrationLogFileExists(githubOrg, $"{teamProject1}-{teamProject1}");
             Helper.AssertMigrationLogFileExists(githubOrg, $"{teamProject2}-{teamProject2}");
         }