Add options for target uploads URL

RELEASENOTES.md

@@ -1 +1 @@
-
+- Add --target-uploads-url option to support GitHub-owned storage uploads to locations other than uploads.github.com.

src/Octoshift/Commands/CreateTeam/CreateTeamCommandBase.cs

@@ -43,7 +43,7 @@ public override CreateTeamCommandHandler BuildHandler(CreateTeamCommandArgs args
         var log = sp.GetRequiredService<OctoLogger>();
         var githubApiFactory = sp.GetRequiredService<ITargetGithubApiFactory>();
 
-        var githubApi = githubApiFactory.Create(args.TargetApiUrl, args.GithubPat);
+        var githubApi = githubApiFactory.Create(args.TargetApiUrl, null, args.GithubPat);
 
         return new CreateTeamCommandHandler(log, githubApi);
     }

src/Octoshift/Commands/DownloadLogs/DownloadLogsCommandBase.cs

@@ -67,7 +67,7 @@ public override DownloadLogsCommandHandler BuildHandler(DownloadLogsCommandArgs
 
         var log = sp.GetRequiredService<OctoLogger>();
         var githubApiFactory = sp.GetRequiredService<ITargetGithubApiFactory>();
-        var githubApi = githubApiFactory.Create(args.GithubApiUrl, args.GithubPat);
+        var githubApi = githubApiFactory.Create(args.GithubApiUrl, null, args.GithubPat);
         var httpDownloadServiceFactory = sp.GetRequiredService<HttpDownloadServiceFactory>();
         var httpDownloadService = httpDownloadServiceFactory.CreateDefaultWithRedirects();
         var retryPolicy = sp.GetRequiredService<RetryPolicy>();

src/Octoshift/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommandBase.cs

@@ -56,7 +56,7 @@ public override GenerateMannequinCsvCommandHandler BuildHandler(GenerateMannequi
 
         var log = sp.GetRequiredService<OctoLogger>();
         var githubApiFactory = sp.GetRequiredService<ITargetGithubApiFactory>();
-        var githubApi = githubApiFactory.Create(args.TargetApiUrl, args.GithubPat);
+        var githubApi = githubApiFactory.Create(args.TargetApiUrl, null, args.GithubPat);
 
         return new GenerateMannequinCsvCommandHandler(log, githubApi);
     }

src/Octoshift/Commands/GrantMigratorRole/GrantMigratorRoleCommandBase.cs

@@ -53,7 +53,7 @@ public override GrantMigratorRoleCommandHandler BuildHandler(GrantMigratorRoleCo
         var log = sp.GetRequiredService<OctoLogger>();
         var githubApiFactory = sp.GetRequiredService<ITargetGithubApiFactory>();
         var apiUrl = args.TargetApiUrl ?? args.GhesApiUrl;
-        var githubApi = githubApiFactory.Create(apiUrl, args.GithubPat);
+        var githubApi = githubApiFactory.Create(apiUrl, null, args.GithubPat);
 
         return new GrantMigratorRoleCommandHandler(log, githubApi);
     }

src/Octoshift/Commands/ReclaimMannequin/ReclaimMannequinCommandBase.cs

@@ -88,7 +88,7 @@ public override ReclaimMannequinCommandHandler BuildHandler(ReclaimMannequinComm
 
         var log = sp.GetRequiredService<OctoLogger>();
         var githubApiFactory = sp.GetRequiredService<ITargetGithubApiFactory>();
-        var githubApi = githubApiFactory.Create(args.TargetApiUrl, args.GithubPat);
+        var githubApi = githubApiFactory.Create(args.TargetApiUrl, null, args.GithubPat);
         var reclaimService = new ReclaimService(githubApi, log);
         var confirmationService = sp.GetRequiredService<ConfirmationService>();
 

src/Octoshift/Commands/RevokeMigratorRole/RevokeMigratorRoleCommandBase.cs

@@ -42,7 +42,7 @@ public override RevokeMigratorRoleCommandHandler BuildHandler(RevokeMigratorRole
         var log = sp.GetRequiredService<OctoLogger>();
         var githubApiFactory = sp.GetRequiredService<ITargetGithubApiFactory>();
         var apiUrl = args.TargetApiUrl ?? args.GhesApiUrl;
-        var githubApi = githubApiFactory.Create(apiUrl, args.GithubPat);
+        var githubApi = githubApiFactory.Create(apiUrl, null, args.GithubPat);
 
         return new RevokeMigratorRoleCommandHandler(log, githubApi);
     }

src/Octoshift/Contracts/ISourceGithubApiFactory.cs

@@ -4,6 +4,6 @@ namespace OctoshiftCLI.Contracts;
 
 public interface ISourceGithubApiFactory
 {
-    GithubApi Create(string apiUrl = null, string sourcePersonalAccessToken = null);
-    GithubApi CreateClientNoSsl(string apiUrl = null, string sourcePersonalAccessToken = null);
+    GithubApi Create(string apiUrl = null, string uploadsUrl = null, string sourcePersonalAccessToken = null);
+    GithubApi CreateClientNoSsl(string apiUrl = null, string uploadsUrl = null, string sourcePersonalAccessToken = null);
 }

src/Octoshift/Contracts/ITargetGithubApiFactory.cs

@@ -4,5 +4,5 @@ namespace OctoshiftCLI.Contracts;
 
 public interface ITargetGithubApiFactory
 {
-    GithubApi Create(string apiUrl = null, string targetPersonalAccessToken = null);
+    GithubApi Create(string apiUrl = null, string uploadsUrl = null, string targetPersonalAccessToken = null);
 }

src/Octoshift/Factories/GithubApiFactory.cs

@@ -7,6 +7,7 @@ namespace OctoshiftCLI.Factories;
 public sealed class GithubApiFactory : ISourceGithubApiFactory, ITargetGithubApiFactory
 {
     private const string DEFAULT_API_URL = "https://api.github.com";
+    private const string DEFAULT_UPLOADS_URL = "https://uploads.github.com";
 
     private readonly OctoLogger _octoLogger;
     private readonly IHttpClientFactory _clientFactory;
@@ -25,30 +26,33 @@ public GithubApiFactory(OctoLogger octoLogger, IHttpClientFactory clientFactory,
         _versionProvider = versionProvider;
     }
 
-    GithubApi ISourceGithubApiFactory.Create(string apiUrl, string sourcePersonalAccessToken)
+    GithubApi ISourceGithubApiFactory.Create(string apiUrl, string uploadsUrl, string sourcePersonalAccessToken)
     {
         apiUrl ??= DEFAULT_API_URL;
+        uploadsUrl ??= DEFAULT_UPLOADS_URL;
         sourcePersonalAccessToken ??= _environmentVariableProvider.SourceGithubPersonalAccessToken();
         var githubClient = new GithubClient(_octoLogger, _clientFactory.CreateClient("Default"), _versionProvider, _retryPolicy, _dateTimeProvider, sourcePersonalAccessToken);
-        var multipartUploader = new ArchiveUploader(githubClient, _octoLogger, _retryPolicy);
+        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy);
         return new GithubApi(githubClient, apiUrl, _retryPolicy, multipartUploader);
     }
 
-    GithubApi ISourceGithubApiFactory.CreateClientNoSsl(string apiUrl, string sourcePersonalAccessToken)
+    GithubApi ISourceGithubApiFactory.CreateClientNoSsl(string apiUrl, string uploadsUrl, string sourcePersonalAccessToken)
     {
         apiUrl ??= DEFAULT_API_URL;
+        uploadsUrl ??= DEFAULT_UPLOADS_URL;
         sourcePersonalAccessToken ??= _environmentVariableProvider.SourceGithubPersonalAccessToken();
         var githubClient = new GithubClient(_octoLogger, _clientFactory.CreateClient("NoSSL"), _versionProvider, _retryPolicy, _dateTimeProvider, sourcePersonalAccessToken);
-        var multipartUploader = new ArchiveUploader(githubClient, _octoLogger, _retryPolicy);
+        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy);
         return new GithubApi(githubClient, apiUrl, _retryPolicy, multipartUploader);
     }
 
-    GithubApi ITargetGithubApiFactory.Create(string apiUrl, string targetPersonalAccessToken)
+    GithubApi ITargetGithubApiFactory.Create(string apiUrl, string uploadsUrl, string targetPersonalAccessToken)
     {
         apiUrl ??= DEFAULT_API_URL;
+        uploadsUrl ??= DEFAULT_UPLOADS_URL;
         targetPersonalAccessToken ??= _environmentVariableProvider.TargetGithubPersonalAccessToken();
         var githubClient = new GithubClient(_octoLogger, _clientFactory.CreateClient("Default"), _versionProvider, _retryPolicy, _dateTimeProvider, targetPersonalAccessToken);
-        var multipartUploader = new ArchiveUploader(githubClient, _octoLogger, _retryPolicy);
+        var multipartUploader = new ArchiveUploader(githubClient, uploadsUrl, _octoLogger, _retryPolicy);
         return new GithubApi(githubClient, apiUrl, _retryPolicy, multipartUploader);
     }
 }

src/Octoshift/Services/ArchiveUploader.cs

@@ -13,15 +13,15 @@ namespace OctoshiftCLI.Services;
 public class ArchiveUploader
 {
     private readonly GithubClient _client;
+    private readonly string _uploadsUrl;
     private readonly OctoLogger _log;
     internal int _streamSizeLimit = 100 * 1024 * 1024; // 100 MiB
     private readonly RetryPolicy _retryPolicy;
 
-    private const string BASE_URL = "https://uploads.github.com";
-
-    public ArchiveUploader(GithubClient client, OctoLogger log, RetryPolicy retryPolicy)
+    public ArchiveUploader(GithubClient client, string uploadsUrl, OctoLogger log, RetryPolicy retryPolicy)
     {
         _client = client;
+        _uploadsUrl = uploadsUrl;
         _log = log;
         _retryPolicy = retryPolicy;
     }
@@ -41,14 +41,14 @@ public virtual async Task<string> Upload(Stream archiveContent, string archiveNa
 
         if (isMultipart)
         {
-            var url = $"{BASE_URL}/organizations/{orgDatabaseId.EscapeDataString()}/gei/archive/blobs/uploads";
+            var url = $"{_uploadsUrl}/organizations/{orgDatabaseId.EscapeDataString()}/gei/archive/blobs/uploads";
 
             response = await UploadMultipart(archiveContent, archiveName, url);
             return response;
         }
         else
         {
-            var url = $"{BASE_URL}/organizations/{orgDatabaseId.EscapeDataString()}/gei/archive?name={archiveName.EscapeDataString()}";
+            var url = $"{_uploadsUrl}/organizations/{orgDatabaseId.EscapeDataString()}/gei/archive?name={archiveName.EscapeDataString()}";
 
             response = await _retryPolicy.Retry(async () => await _client.PostAsync(url, streamContent));
             var data = JObject.Parse(response);
@@ -155,7 +155,7 @@ private Uri GetNextUrl(IEnumerable<KeyValuePair<string, IEnumerable<string>>> he
             var locationValue = locationHeader.Value.FirstOrDefault();
             if (locationValue.HasValue())
             {
-                return new Uri(new Uri(BASE_URL), locationValue);
+                return new Uri(new Uri(_uploadsUrl), locationValue);
             }
         }
         throw new OctoshiftCliException("Location header is missing in the response, unable to retrieve next URL for multipart upload.");

src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs

@@ -16,6 +16,7 @@ public sealed class BbsToGithub : IDisposable
 {
     private const string SSH_KEY_FILE = "ssh_key.pem";
     private const string AWS_REGION = "us-east-1";
+    private const string UPLOADS_URL = "https://uploads.github.com";
 
     private readonly ITestOutputHelper _output;
     private readonly OctoLogger _logger;
@@ -60,7 +61,7 @@ public BbsToGithub(ITestOutputHelper output)
         _targetGithubHttpClient = new HttpClient();
         _targetGithubClient = new GithubClient(_logger, _targetGithubHttpClient, new VersionChecker(_versionClient, _logger), new RetryPolicy(_logger), new DateTimeProvider(), targetGithubToken);
         var retryPolicy = new RetryPolicy(_logger);
-        _archiveUploader = new ArchiveUploader(_targetGithubClient, _logger, retryPolicy);
+        _archiveUploader = new ArchiveUploader(_targetGithubClient, UPLOADS_URL, _logger, retryPolicy);
         _targetGithubApi = new GithubApi(_targetGithubClient, "https://api.github.com", new RetryPolicy(_logger), _archiveUploader);
 
         _blobServiceClient = new BlobServiceClient(_azureStorageConnectionString);

src/OctoshiftCLI.IntegrationTests/GhesToGithub.cs

@@ -13,6 +13,7 @@ namespace OctoshiftCLI.IntegrationTests;
 public sealed class GhesToGithub : IDisposable
 {
     private const string GHES_API_URL = "https://octoshift-ghe.westus2.cloudapp.azure.com/api/v3";
+    private const string UPLOADS_URL = "https://uploads.github.com";
 
     private readonly ITestOutputHelper _output;
     private readonly TestHelper _targetHelper;
@@ -48,7 +49,7 @@ public GhesToGithub(ITestOutputHelper output)
 
         _versionClient = new HttpClient();
         var retryPolicy = new RetryPolicy(logger);
-        _archiveUploader = new ArchiveUploader(_targetGithubClient, logger, retryPolicy);
+        _archiveUploader = new ArchiveUploader(_targetGithubClient, UPLOADS_URL, logger, retryPolicy);
 
         _sourceGithubHttpClient = new HttpClient();
         _sourceGithubClient = new GithubClient(logger, _sourceGithubHttpClient, new VersionChecker(_versionClient, logger), new RetryPolicy(logger), new DateTimeProvider(), sourceGithubToken);

src/OctoshiftCLI.Tests/Octoshift/Commands/AbortMigration/AbortMigrationCommandBaseTests.cs

@@ -39,7 +39,7 @@ public void It_Uses_The_TargetApiUrl_When_Provided()
 
         _command.BuildHandler(args, _serviceProvider);
 
-        _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, It.IsAny<string>()));
+        _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, null, It.IsAny<string>()));
     }
 }
 

src/OctoshiftCLI.Tests/Octoshift/Commands/CreateTeam/CreateTeamCommandBaseTests.cs

@@ -40,7 +40,7 @@ public void It_Uses_Github_Source_Pat_When_Provided()
 
         _command.BuildHandler(args, _serviceProvider);
 
-        _mockGithubApiFactory.Verify(m => m.Create(It.IsAny<string>(), args.GithubPat), Times.Once);
+        _mockGithubApiFactory.Verify(m => m.Create(It.IsAny<string>(), It.IsAny<string>(), args.GithubPat), Times.Once);
     }
 
     [Fact]
@@ -57,6 +57,6 @@ public void It_Uses_Target_Api_Url_When_Provided()
 
         _command.BuildHandler(args, _serviceProvider);
 
-        _mockGithubApiFactory.Verify(m => m.Create(args.TargetApiUrl, null), Times.Once);
+        _mockGithubApiFactory.Verify(m => m.Create(args.TargetApiUrl, It.IsAny<string>(), null), Times.Once);
     }
 }

src/OctoshiftCLI.Tests/Octoshift/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommandBaseTests.cs

@@ -39,6 +39,6 @@ public void It_Uses_The_TargetApiUrl_When_Provided()
 
         _command.BuildHandler(args, _serviceProvider);
 
-        _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, null));
+        _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, null, null));
     }
 }

src/OctoshiftCLI.Tests/Octoshift/Commands/GrantMigratorRole/GrantMigratorRoleCommandBaseTests.cs

@@ -41,7 +41,7 @@ public void It_Uses_Github_Target_Pat_When_Provided()
 
         _command.BuildHandler(args, _serviceProvider);
 
-        _mockGithubApiFactory.Verify(m => m.Create(It.IsAny<string>(), githubPat));
+        _mockGithubApiFactory.Verify(m => m.Create(It.IsAny<string>(), It.IsAny<string>(), githubPat));
     }
 
     [Fact]
@@ -59,7 +59,7 @@ public void It_Uses_The_GhesApiUrl_When_Provided()
 
         _command.BuildHandler(args, _serviceProvider);
 
-        _mockGithubApiFactory.Verify(m => m.Create(ghesApiUrl, null));
+        _mockGithubApiFactory.Verify(m => m.Create(ghesApiUrl, It.IsAny<string>(), null));
     }
 
     [Fact]
@@ -77,6 +77,6 @@ public void It_Uses_The_TargetApiUrl_When_Provided()
 
         _command.BuildHandler(args, _serviceProvider);
 
-        _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, null));
+        _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, It.IsAny<string>(), null));
     }
 }

src/OctoshiftCLI.Tests/Octoshift/Commands/ReclaimMannequin/ReclaimMannequinCommandBaseTests.cs

@@ -42,6 +42,6 @@ public void It_Uses_The_TargetApiUrl_When_Provided()
 
         _command.BuildHandler(args, _serviceProvider);
 
-        _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, null));
+        _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, null, null));
     }
 }

src/OctoshiftCLI.Tests/Octoshift/Commands/RevokeMigratorRole/RevokeMigratorRoleCommandBaseTests.cs

@@ -41,7 +41,7 @@ public void It_Uses_Github_Target_Pat_When_Provided()
 
         _command.BuildHandler(args, _serviceProvider);
 
-        _mockGithubApiFactory.Verify(m => m.Create(It.IsAny<string>(), githubPat));
+        _mockGithubApiFactory.Verify(m => m.Create(It.IsAny<string>(), It.IsAny<string>(), githubPat));
     }
 
     [Fact]
@@ -59,7 +59,7 @@ public void It_Uses_The_GhesApiUrl_When_Provided()
 
         _command.BuildHandler(args, _serviceProvider);
 
-        _mockGithubApiFactory.Verify(m => m.Create(ghesApiUrl, null));
+        _mockGithubApiFactory.Verify(m => m.Create(ghesApiUrl, It.IsAny<string>(), null));
     }
 
     [Fact]
@@ -79,6 +79,6 @@ public void It_Uses_The_TargetApiUrl_When_Provided()
 
         _command.BuildHandler(args, _serviceProvider);
 
-        _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, null));
+        _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, It.IsAny<string>(), null));
     }
 }

src/OctoshiftCLI.Tests/Octoshift/Commands/WaitForMigration/WaitForMigrationCommandBaseTests.cs

@@ -40,7 +40,7 @@ public void It_Uses_The_TargetApiUrl_When_Provided()
 
         _command.BuildHandler(args, _serviceProvider);
 
-        _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, It.IsAny<string>()));
+        _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, null, It.IsAny<string>()));
     }
 }
 

src/OctoshiftCLI.Tests/Octoshift/Services/ArchiveUploadersTests.cs

@@ -13,6 +13,8 @@ namespace OctoshiftCLI.Tests.Octoshift.Services;
 
 public class ArchiveUploaderTests
 {
+    private const string UPLOADS_URL = "https://uploads.github.com";
+
     private readonly Mock<GithubClient> _githubClientMock;
     private readonly Mock<OctoLogger> _logMock;
     private readonly ArchiveUploader _archiveUploader;
@@ -22,7 +24,7 @@ public ArchiveUploaderTests()
         _logMock = TestHelpers.CreateMock<OctoLogger>();
         _githubClientMock = TestHelpers.CreateMock<GithubClient>();
         var retryPolicy = new RetryPolicy(_logMock.Object) { _httpRetryInterval = 1, _retryInterval = 0 };
-        _archiveUploader = new ArchiveUploader(_githubClientMock.Object, _logMock.Object, retryPolicy);
+        _archiveUploader = new ArchiveUploader(_githubClientMock.Object, UPLOADS_URL, _logMock.Object, retryPolicy);
     }
 
     [Fact]

src/OctoshiftCLI.Tests/ado2gh/Commands/AddTeamToRepo/AddTeamToRepoCommandTests.cs

@@ -59,7 +59,7 @@ public void It_Uses_The_Github_Pat_When_Provided()
 
             _command.BuildHandler(args, _serviceProvider);
 
-            _mockGithubApiFactory.Verify(m => m.Create(null, githubPat));
+            _mockGithubApiFactory.Verify(m => m.Create(null, null, githubPat));
         }
 
         [Fact]
@@ -73,7 +73,7 @@ public async Task Invalid_Role()
             command.SetHandler(async x => await command.BuildHandler(x, _serviceProvider).Handle(x), argsBinder);
             await command.InvokeAsync(args);
 
-            _mockGithubApiFactory.Verify(x => x.Create(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
+            _mockGithubApiFactory.Verify(x => x.Create(It.IsAny<string>(), null, It.IsAny<string>()), Times.Never);
         }
     }
 }

src/OctoshiftCLI.Tests/ado2gh/Commands/ConfigureAutoLinkCommand/ConfigureAutoLinkCommandTests.cs

@@ -57,7 +57,7 @@ public void It_Uses_The_Github_Pat_When_Provided()
 
             _command.BuildHandler(args, _serviceProvider);
 
-            _mockGithubApiFactory.Verify(m => m.Create(null, githubPat));
+            _mockGithubApiFactory.Verify(m => m.Create(null, null, githubPat));
         }
     }
 }

src/OctoshiftCLI.Tests/ado2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs

@@ -70,7 +70,7 @@ public void It_Uses_Github_Pat_When_Provided()
 
             _command.BuildHandler(args, _serviceProvider);
 
-            _mockGithubApiFactory.Verify(m => m.Create(null, githubPat));
+            _mockGithubApiFactory.Verify(m => m.Create(null, It.IsAny<string>(), githubPat));
         }
         [Fact]
         public void It_Uses_Target_Api_Url_When_Provided()
@@ -93,7 +93,7 @@ public void It_Uses_Target_Api_Url_When_Provided()
 
             _command.BuildHandler(args, _serviceProvider);
 
-            _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, githubPat));
+            _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, It.IsAny<string>(), githubPat));
         }
     }
 }