Skip to content

Commit fbd9a24

Browse files
syntheadtamblingArin Ghazarian
authored
Support uploading archives to AWS S3 buckets (#653)
<!-- For the checkboxes below you must check each one to indicate that you either did the relevant task, or considered it and decided there was nothing that needed doing --> Closes #594. Related to github/releases#2532. - [ ] Did you write/update appropriate tests - [ ] Release notes updated (if appropriate) - [ ] Appropriate logging output - [ ] Issue linked - [ ] Docs updated (or issue created) <!-- For docs we should review the docs at: https://docs.github.com/en/early-access/github/migrating-with-github-enterprise-importer and the README.md in this repo If a doc update is required based on the changes in this PR, it is sufficient to create an issue and link to it here. The doc update can be made later/separately. The process to update the docs can be found here: https://github.com/github/docs-early-access#opening-prs The markdown files are here: https://github.com/github/docs-early-access/tree/main/content/github/migrating-with-github-enterprise-importer --> Co-authored-by: Dean Tambling <[email protected]> Co-authored-by: Arin Ghazarian <[email protected]>
1 parent 8e1ef90 commit fbd9a24

18 files changed

+468
-37
lines changed

‎RELEASENOTES.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@
44
- If `create-team` fails when removing the initial team member, it will retry
55
- In v0.25 we started publishing `ado2gh` as an extension to the `gh` CLI. However, we didn't update `ado2gh generate-script` to use the new syntax in the generated migration script. Now it will, and you will need the `gh ado2gh` extension installed in order to run the generated migration script.
66
- Added `--ghes-api-url` as an optional arg to the `grant-migrator-role` and `revoke-migrator-role` commands for both `ado2gh` and `gei`.
7-
- Added `--bbs-shared-home` option to `migrate-repo` and `generate-script` commands for `bbs2gh`. This provides support for BBS servers with a non default directory path.
7+
- Added `--bbs-shared-home` option to `migrate-repo` and `generate-script` commands for `bbs2gh`. This provides support for BBS servers with a non default directory path.
8+
- Added AWS support for archive uploads to `gei` and `bbs2gh`, using the `--aws-bucket-name`, `--aws-access-key` and `--aws-secret-key` arguments.

‎src/Octoshift/AwsApi.cs

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
using System;
2+
using System.IO;
3+
using System.Threading.Tasks;
4+
using Amazon;
5+
using Amazon.S3;
6+
using Amazon.S3.Model;
7+
using Amazon.S3.Transfer;
8+
9+
namespace OctoshiftCLI;
10+
11+
public class AwsApi : IDisposable
12+
{
13+
private const int AUTHORIZATION_TIMEOUT_IN_HOURS = 24;
14+
private static readonly RegionEndpoint RegionEndpoint = RegionEndpoint.USEast1;
15+
16+
private readonly ITransferUtility _transferUtility;
17+
18+
#pragma warning disable CA2000
19+
public AwsApi(string awsAccessKey, string awsSecretKey) : this(new TransferUtility(new AmazonS3Client(awsAccessKey, awsSecretKey, RegionEndpoint)))
20+
#pragma warning restore CA2000
21+
{
22+
}
23+
24+
internal AwsApi(ITransferUtility transferUtility) => _transferUtility = transferUtility;
25+
26+
public virtual async Task<string> UploadToBucket(string bucketName, string fileName, string keyName)
27+
{
28+
await _transferUtility.UploadAsync(fileName, bucketName, keyName);
29+
return GetPreSignedUrlForFile(bucketName, keyName);
30+
}
31+
32+
public virtual async Task<string> UploadToBucket(string bucketName, byte[] bytes, string keyName)
33+
{
34+
using var byteStream = new MemoryStream(bytes);
35+
await _transferUtility.UploadAsync(byteStream, bucketName, keyName);
36+
return GetPreSignedUrlForFile(bucketName, keyName);
37+
}
38+
39+
private string GetPreSignedUrlForFile(string bucketName, string keyName)
40+
{
41+
var expires = DateTime.Now.AddHours(AUTHORIZATION_TIMEOUT_IN_HOURS);
42+
43+
var urlRequest = new GetPreSignedUrlRequest
44+
{
45+
BucketName = bucketName,
46+
Key = keyName,
47+
Expires = expires
48+
};
49+
50+
return _transferUtility.S3Client.GetPreSignedURL(urlRequest);
51+
}
52+
53+
protected virtual void Dispose(bool disposing)
54+
{
55+
if (disposing)
56+
{
57+
_transferUtility?.Dispose();
58+
}
59+
}
60+
61+
public void Dispose()
62+
{
63+
Dispose(true);
64+
GC.SuppressFinalize(this);
65+
}
66+
}

‎src/Octoshift/Octoshift.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
</PropertyGroup>
77

88
<ItemGroup>
9+
<PackageReference Include="AWSSDK.S3" Version="3.7.9.52" />
910
<PackageReference Include="Azure.Storage.Blobs" Version="12.13.0" />
1011
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="3.11.0" />
1112
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />

‎src/OctoshiftCLI.Tests/AwsApiTests.cs

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
using System.IO;
2+
using System.Text;
3+
using System.Threading;
4+
using System.Threading.Tasks;
5+
using Amazon.S3;
6+
using Amazon.S3.Model;
7+
using Amazon.S3.Transfer;
8+
using FluentAssertions;
9+
using Moq;
10+
using Xunit;
11+
12+
13+
namespace OctoshiftCLI.Tests;
14+
15+
public class AwsApiTests
16+
{
17+
[Fact]
18+
public async Task UploadFileToBucket_Should_Succeed()
19+
{
20+
// Arrange
21+
var bucketName = "bucket";
22+
var fileName = "file.zip";
23+
var keyName = "key";
24+
var url = "http://example.com/file.zip";
25+
26+
var transferUtility = new Mock<ITransferUtility>();
27+
var s3Client = new Mock<IAmazonS3>();
28+
29+
s3Client.Setup(m => m.GetPreSignedURL(It.IsAny<GetPreSignedUrlRequest>())).Returns(url);
30+
transferUtility.Setup(m => m.S3Client).Returns(s3Client.Object);
31+
using var awsApi = new AwsApi(transferUtility.Object);
32+
33+
var result = await awsApi.UploadToBucket(bucketName, fileName, keyName);
34+
35+
// Assert
36+
result.Should().Be(url);
37+
transferUtility.Verify(m => m.UploadAsync(fileName, bucketName, keyName, It.IsAny<CancellationToken>()));
38+
}
39+
40+
[Fact]
41+
public async Task UploadToBucket_Uploads_Byte_Array()
42+
{
43+
// Arrange
44+
var bucketName = "bucket";
45+
var bytes = Encoding.ASCII.GetBytes("here are some bytes");
46+
var keyName = "key";
47+
var url = "http://example.com/file.zip";
48+
49+
var transferUtility = new Mock<ITransferUtility>();
50+
var s3Client = new Mock<IAmazonS3>();
51+
52+
s3Client.Setup(m => m.GetPreSignedURL(It.IsAny<GetPreSignedUrlRequest>())).Returns(url);
53+
transferUtility.Setup(m => m.S3Client).Returns(s3Client.Object);
54+
using var awsApi = new AwsApi(transferUtility.Object);
55+
56+
var result = await awsApi.UploadToBucket(bucketName, bytes, keyName);
57+
58+
// Assert
59+
result.Should().Be(url);
60+
transferUtility.Verify(m => m.UploadAsync(It.IsAny<MemoryStream>(), bucketName, keyName, It.IsAny<CancellationToken>()));
61+
}
62+
}

‎src/OctoshiftCLI.Tests/bbs2gh/Commands/MigrateRepoCommandTests.cs

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -46,29 +46,32 @@ public MigrateRepoCommandTests()
4646
[Fact]
4747
public void Should_Have_Options()
4848
{
49-
_command.Should().NotBeNull();
50-
_command.Name.Should().Be("migrate-repo");
51-
_command.Options.Count.Should().Be(19);
52-
53-
TestHelpers.VerifyCommandOption(_command.Options, "bbs-server-url", false);
54-
TestHelpers.VerifyCommandOption(_command.Options, "bbs-project", false);
55-
TestHelpers.VerifyCommandOption(_command.Options, "bbs-repo", false);
56-
TestHelpers.VerifyCommandOption(_command.Options, "bbs-username", false);
57-
TestHelpers.VerifyCommandOption(_command.Options, "bbs-password", false);
58-
TestHelpers.VerifyCommandOption(_command.Options, "bbs-shared-home", false);
59-
TestHelpers.VerifyCommandOption(_command.Options, "archive-url", false);
60-
TestHelpers.VerifyCommandOption(_command.Options, "archive-path", false);
61-
TestHelpers.VerifyCommandOption(_command.Options, "azure-storage-connection-string", false);
62-
TestHelpers.VerifyCommandOption(_command.Options, "github-org", false);
63-
TestHelpers.VerifyCommandOption(_command.Options, "github-repo", false);
64-
TestHelpers.VerifyCommandOption(_command.Options, "github-pat", false);
65-
TestHelpers.VerifyCommandOption(_command.Options, "ssh-user", false);
66-
TestHelpers.VerifyCommandOption(_command.Options, "ssh-private-key", false);
67-
TestHelpers.VerifyCommandOption(_command.Options, "ssh-port", false);
68-
TestHelpers.VerifyCommandOption(_command.Options, "smb-user", false, true);
69-
TestHelpers.VerifyCommandOption(_command.Options, "smb-password", false, true);
70-
TestHelpers.VerifyCommandOption(_command.Options, "wait", false);
71-
TestHelpers.VerifyCommandOption(_command.Options, "verbose", false);
49+
var command = new MigrateRepoCommand();
50+
command.Should().NotBeNull();
51+
command.Name.Should().Be("migrate-repo");
52+
command.Options.Count.Should().Be(22);
53+
54+
TestHelpers.VerifyCommandOption(command.Options, "bbs-server-url", false);
55+
TestHelpers.VerifyCommandOption(command.Options, "bbs-project", false);
56+
TestHelpers.VerifyCommandOption(command.Options, "bbs-repo", false);
57+
TestHelpers.VerifyCommandOption(command.Options, "bbs-username", false);
58+
TestHelpers.VerifyCommandOption(command.Options, "bbs-password", false);
59+
TestHelpers.VerifyCommandOption(command.Options, "archive-url", false);
60+
TestHelpers.VerifyCommandOption(command.Options, "archive-path", false);
61+
TestHelpers.VerifyCommandOption(command.Options, "azure-storage-connection-string", false);
62+
TestHelpers.VerifyCommandOption(command.Options, "aws-bucket-name", false);
63+
TestHelpers.VerifyCommandOption(command.Options, "aws-access-key", false);
64+
TestHelpers.VerifyCommandOption(command.Options, "aws-secret-key", false);
65+
TestHelpers.VerifyCommandOption(command.Options, "github-org", false);
66+
TestHelpers.VerifyCommandOption(command.Options, "github-repo", false);
67+
TestHelpers.VerifyCommandOption(command.Options, "github-pat", false);
68+
TestHelpers.VerifyCommandOption(command.Options, "ssh-user", false);
69+
TestHelpers.VerifyCommandOption(command.Options, "ssh-private-key", false);
70+
TestHelpers.VerifyCommandOption(command.Options, "ssh-port", false);
71+
TestHelpers.VerifyCommandOption(command.Options, "smb-user", false, true);
72+
TestHelpers.VerifyCommandOption(command.Options, "smb-password", false, true);
73+
TestHelpers.VerifyCommandOption(command.Options, "wait", false);
74+
TestHelpers.VerifyCommandOption(command.Options, "verbose", false);
7275
}
7376

7477
[Fact]

‎src/OctoshiftCLI.Tests/bbs2gh/Handlers/MigrateRepoCommandHandlerTests.cs

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ public class MigrateRepoCommandHandlerTests
1515
private readonly Mock<GithubApi> _mockGithubApi = TestHelpers.CreateMock<GithubApi>();
1616
private readonly Mock<BbsApi> _mockBbsApi = TestHelpers.CreateMock<BbsApi>();
1717
private readonly Mock<AzureApi> _mockAzureApi = TestHelpers.CreateMock<AzureApi>();
18+
private readonly Mock<AwsApi> _mockAwsApi = TestHelpers.CreateMock<AwsApi>();
1819
private readonly Mock<OctoLogger> _mockOctoLogger = TestHelpers.CreateMock<OctoLogger>();
1920
private readonly Mock<EnvironmentVariableProvider> _mockEnvironmentVariableProvider = TestHelpers.CreateMock<EnvironmentVariableProvider>();
2021
private readonly Mock<IBbsArchiveDownloader> _mockBbsArchiveDownloader = new();
@@ -27,6 +28,9 @@ public class MigrateRepoCommandHandlerTests
2728
private const string GITHUB_ORG = "target-org";
2829
private const string GITHUB_REPO = "target-repo";
2930
private const string GITHUB_PAT = "github pat";
31+
private const string AWS_BUCKET_NAME = "aws-bucket-name";
32+
private const string AWS_ACCESS_KEY = "aws-access-key";
33+
private const string AWS_SECRET_KEY = "aws-secret-key";
3034

3135
private const string BBS_HOST = "our-bbs-server.com";
3236
private const string BBS_SERVER_URL = $"https://{BBS_HOST}";
@@ -52,6 +56,7 @@ public MigrateRepoCommandHandlerTests()
5256
_mockEnvironmentVariableProvider.Object,
5357
_mockBbsArchiveDownloader.Object,
5458
_mockAzureApi.Object,
59+
_mockAwsApi.Object,
5560
_mockFileSystemProvider.Object
5661
);
5762
}
@@ -328,5 +333,44 @@ public async Task Errors_If_BbsServer_Url_And_Archive_Path_Are_Passed()
328333
// Assert
329334
await _handler.Invoking(x => x.Handle(args)).Should().ThrowExactlyAsync<OctoshiftCliException>();
330335
}
336+
337+
[Fact]
338+
public async Task Uses_Aws_If_Credentials_Are_Passed()
339+
{
340+
// Arrange
341+
_mockEnvironmentVariableProvider.Setup(m => m.GithubPersonalAccessToken()).Returns(GITHUB_PAT);
342+
343+
_mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID);
344+
_mockGithubApi.Setup(x => x.CreateBbsMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID);
345+
_mockGithubApi
346+
.Setup(x => x.StartBbsMigration(MIGRATION_SOURCE_ID, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL).Result)
347+
.Returns(MIGRATION_ID);
348+
349+
_mockAwsApi.Setup(x => x.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny<string>())).ReturnsAsync(ARCHIVE_URL);
350+
351+
// Act
352+
var args = new MigrateRepoCommandArgs
353+
{
354+
GithubOrg = GITHUB_ORG,
355+
GithubRepo = GITHUB_REPO,
356+
ArchivePath = ARCHIVE_PATH,
357+
AwsAccessKey = AWS_ACCESS_KEY,
358+
AwsSecretKey = AWS_SECRET_KEY,
359+
AwsBucketName = AWS_BUCKET_NAME
360+
};
361+
362+
await _handler.Handle(args);
363+
364+
// Assert
365+
_mockGithubApi.Verify(m => m.StartBbsMigration(
366+
MIGRATION_SOURCE_ID,
367+
GITHUB_ORG_ID,
368+
GITHUB_REPO,
369+
GITHUB_PAT,
370+
ARCHIVE_URL
371+
));
372+
373+
_mockAwsApi.Verify(m => m.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny<string>()));
374+
}
331375
}
332376
}

‎src/OctoshiftCLI.Tests/gei/Commands/MigrateRepoCommandTests.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,11 @@ public class MigrateRepoCommandTests
99
[Fact]
1010
public void Should_Have_Options()
1111
{
12-
var command = new MigrateRepoCommand(null, null, null, null, null);
12+
var command = new MigrateRepoCommand(null, null, null, null, null, null);
1313

1414
command.Should().NotBeNull();
1515
command.Name.Should().Be("migrate-repo");
16-
command.Options.Count.Should().Be(20);
16+
command.Options.Count.Should().Be(23);
1717

1818
TestHelpers.VerifyCommandOption(command.Options, "github-source-org", false);
1919
TestHelpers.VerifyCommandOption(command.Options, "ado-server-url", false, true);
@@ -25,6 +25,9 @@ public void Should_Have_Options()
2525
TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false);
2626
TestHelpers.VerifyCommandOption(command.Options, "ghes-api-url", false);
2727
TestHelpers.VerifyCommandOption(command.Options, "azure-storage-connection-string", false);
28+
TestHelpers.VerifyCommandOption(command.Options, "aws-bucket-name", false);
29+
TestHelpers.VerifyCommandOption(command.Options, "aws-access-key", false);
30+
TestHelpers.VerifyCommandOption(command.Options, "aws-secret-key", false);
2831
TestHelpers.VerifyCommandOption(command.Options, "no-ssl-verify", false);
2932
TestHelpers.VerifyCommandOption(command.Options, "skip-releases", false);
3033
TestHelpers.VerifyCommandOption(command.Options, "git-archive-url", false, true);

0 commit comments

Comments
 (0)