Skip to content

Commit d5bcc2b

Browse files
authored
Merge pull request #1055 from github/timrogers/require-aws-region
Require the AWS region to be specified if using AWS S3 for blob storage
2 parents e455609 + 40ef791 commit d5bcc2b

File tree

13 files changed

+26
-48
lines changed

13 files changed

+26
-48
lines changed

RELEASENOTES.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
- __BREAKING CHANGE:__ Require the AWS region to always be specified with the `--aws-region` argument or `AWS_REGION` environment variable if using AWS S3 for blob storage. Previously, this was optional (with a warning) if you weren't specifying an AWS session token.
12
- __BREAKING CHANGE:__ Drop support for deprecated `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` environment variables in `gh gei` and `gh bbs2gh`. The AWS S3 credentials can now only be configured using the industry-standard `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` variables or command line arguments.
23
- __BREAKING CHANGE__: Require the Bitbucket Server URL, project key and repo to always be provided for `bbs2gh migrate-repo`, even if using the upload-and-migrate (`--archive-path`) or migrate-only (`--archive-url`) flows
34
- Increase timeouts in archive uploads to AWS to prevent timeouts during large uploads

src/Octoshift/Services/AwsApi.cs

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,27 +13,22 @@ namespace OctoshiftCLI.Services;
1313
public class AwsApi : IDisposable
1414
{
1515
private const int AUTHORIZATION_TIMEOUT_IN_HOURS = 48;
16-
private static readonly RegionEndpoint DefaultRegionEndpoint = RegionEndpoint.USEast1;
1716

1817
private readonly ITransferUtility _transferUtility;
1918

19+
public AwsApi(ITransferUtility transferUtility) => _transferUtility = transferUtility;
20+
2021
#pragma warning disable CA2000
2122
public AwsApi(string awsAccessKeyId, string awsSecretAccessKey, string awsRegion = null, string awsSessionToken = null)
2223
: this(new TransferUtility(BuildAmazonS3Client(awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken)))
2324
#pragma warning restore CA2000
2425
{
2526
}
2627

27-
internal AwsApi(ITransferUtility transferUtility) => _transferUtility = transferUtility;
28-
2928
private static AmazonS3Client BuildAmazonS3Client(string awsAccessKeyId, string awsSecretAccessKey, string awsRegion, string awsSessionToken)
3029
{
31-
var regionEndpoint = DefaultRegionEndpoint;
32-
if (awsRegion.HasValue())
33-
{
34-
regionEndpoint = GetRegionEndpoint(awsRegion);
35-
AWSConfigsS3.UseSignatureVersion4 = true;
36-
}
30+
var regionEndpoint = awsRegion.IsNullOrWhiteSpace() ? null : GetRegionEndpoint(awsRegion);
31+
AWSConfigsS3.UseSignatureVersion4 = true;
3732

3833
var creds = awsSessionToken.HasValue()
3934
? (AWSCredentials)new SessionAWSCredentials(awsAccessKeyId, awsSecretAccessKey, awsSessionToken)

src/OctoshiftCLI.IntegrationTests/BbsToGithub.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ public sealed class BbsToGithub : IDisposable
1717

1818
private const string SSH_KEY_FILE = "ssh_key.pem";
1919
private const string AWS_BUCKET_NAME = "github-dev";
20+
private const string AWS_REGION = "us-east-1";
2021

2122
private readonly ITestOutputHelper _output;
2223
private readonly OctoLogger _logger;
@@ -115,7 +116,7 @@ await retryPolicy.Retry(async () =>
115116
{
116117
_tokens.Add("AWS_ACCESS_KEY_ID", Environment.GetEnvironmentVariable("AWS_ACCESS_KEY_ID"));
117118
_tokens.Add("AWS_SECRET_ACCESS_KEY", Environment.GetEnvironmentVariable("AWS_SECRET_ACCESS_KEY"));
118-
archiveUploadOptions = $" --aws-bucket-name {AWS_BUCKET_NAME}";
119+
archiveUploadOptions = $" --aws-bucket-name {AWS_BUCKET_NAME} --aws-region {AWS_REGION}";
119120
}
120121

121122
await _targetHelper.RunBbsCliMigration(

src/OctoshiftCLI.Tests/bbs2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ public class MigrateRepoCommandHandlerTests
3434
private const string AWS_ACCESS_KEY_ID = "aws-access-key-id";
3535
private const string AWS_SECRET_ACCESS_KEY = "aws-secret-access-key";
3636
private const string AWS_SESSION_TOKEN = "aws-session-token";
37+
private const string AWS_REGION = "eu-west-1";
3738
private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string";
3839

3940
private const string BBS_HOST = "our-bbs-server.com";
@@ -214,6 +215,7 @@ public async Task Happy_Path_Generate_Archive_Ssh_Download_Aws_Upload_And_Ingest
214215
AwsBucketName = AWS_BUCKET_NAME,
215216
AwsAccessKey = AWS_ACCESS_KEY_ID,
216217
AwsSecretKey = AWS_SECRET_ACCESS_KEY,
218+
AwsRegion = AWS_REGION,
217219
GithubOrg = GITHUB_ORG,
218220
GithubRepo = GITHUB_REPO,
219221
GithubPat = GITHUB_PAT,
@@ -672,6 +674,7 @@ public async Task Uses_Aws_If_Credentials_Are_Passed()
672674
AwsAccessKey = AWS_ACCESS_KEY_ID,
673675
AwsSecretKey = AWS_SECRET_ACCESS_KEY,
674676
AwsBucketName = AWS_BUCKET_NAME,
677+
AwsRegion = AWS_REGION,
675678
QueueOnly = true,
676679
};
677680

@@ -755,7 +758,7 @@ await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
755758
}
756759

757760
[Fact]
758-
public async Task It_Throws_When_Aws_Session_Token_Is_Provided_But_Aws_Region_Is_Not()
761+
public async Task It_Throws_When_Aws_Bucket_Name_Is_Provided_But_No_Aws_Region()
759762
{
760763
await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
761764
{
@@ -769,7 +772,7 @@ await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
769772
}))
770773
.Should()
771774
.ThrowAsync<OctoshiftCliException>()
772-
.WithMessage("*--aws-region*AWS_REGION*--aws-session-token*AWS_SESSION_TOKEN*");
775+
.WithMessage("Either --aws-region or AWS_REGION environment variable must be set.");
773776
}
774777

775778
[Fact]

src/OctoshiftCLI.Tests/gei/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1119,6 +1119,7 @@ public async Task It_Uses_Aws_If_Arguments_Are_Included()
11191119
var awsAccessKeyId = "awsAccessKeyId";
11201120
var awsSecretAccessKey = "awsSecretAccessKey";
11211121
var awsBucketName = "awsBucketName";
1122+
var awsRegion = "eu-west-1";
11221123
var archiveUrl = $"https://s3.amazonaws.com/{awsBucketName}/archive.tar";
11231124

11241125
_mockTargetGithubApi.Setup(x => x.GetOrganizationId(TARGET_ORG).Result).Returns(githubOrgId);
@@ -1181,6 +1182,7 @@ public async Task It_Uses_Aws_If_Arguments_Are_Included()
11811182
AwsBucketName = awsBucketName,
11821183
AwsAccessKey = awsAccessKeyId,
11831184
AwsSecretKey = awsSecretAccessKey,
1185+
AwsRegion = awsRegion,
11841186
Wait = true
11851187
};
11861188

@@ -1250,7 +1252,7 @@ await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
12501252
}
12511253

12521254
[Fact]
1253-
public async Task Ghes_When_Aws_Session_Token_Is_Provided_But_No_Aws_Region_Throws()
1255+
public async Task Ghes_When_Aws_Bucket_Name_Is_Provided_But_No_Aws_Region_Throws()
12541256
{
12551257
_mockGhesVersionChecker.Setup(m => m.AreBlobCredentialsRequired(GHES_API_URL)).ReturnsAsync(true);
12561258

@@ -1268,7 +1270,7 @@ await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs
12681270
}))
12691271
.Should()
12701272
.ThrowAsync<OctoshiftCliException>()
1271-
.WithMessage("*--aws-region*AWS_REGION*--aws-session-token*AWS_SESSION_TOKEN*");
1273+
.WithMessage("Either --aws-region or AWS_REGION environment variable must be set.");
12721274
}
12731275

12741276
[Fact]

src/bbs2gh/Commands/GenerateScript/GenerateScriptCommand.cs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,7 @@ public GenerateScriptCommand() : base(
106106
public Option<string> AwsRegion { get; } = new(
107107
name: "--aws-region",
108108
description: "If using AWS, the AWS region. If not provided, it will be read from AWS_REGION environment variable. " +
109-
"Defaults to us-east-1 if neither the argument nor the environment variable is set. " +
110-
"In a future release, you will be required to set an AWS region if using AWS S3 as your blob storage provider.");
109+
"Required if using AWS.");
111110

112111
public Option<bool> Verbose { get; } = new("--verbose");
113112

src/bbs2gh/Commands/MigrateRepo/MigrateRepoCommand.cs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,7 @@ public MigrateRepoCommand() : base(
116116
public Option<string> AwsRegion { get; } = new(
117117
name: "--aws-region",
118118
description: "If using AWS, the AWS region. If not provided, it will be read from AWS_REGION environment variable. " +
119-
"Defaults to us-east-1 if neither the argument nor the environment variable is set. " +
120-
"In a future release, you will be required to set an AWS region if using AWS S3 as your blob storage provider.");
119+
"Required if using AWS.");
121120

122121
public Option<string> GithubOrg { get; } = new("--github-org");
123122

src/bbs2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -244,9 +244,6 @@ private async Task ImportArchive(MigrateRepoCommandArgs args, string archiveUrl
244244

245245
private string GetAwsRegion(MigrateRepoCommandArgs args) => args.AwsRegion.HasValue() ? args.AwsRegion : _environmentVariableProvider.AwsRegion(false);
246246

247-
private string GetAwsSessionToken(MigrateRepoCommandArgs args) =>
248-
args.AwsSessionToken.HasValue() ? args.AwsSessionToken : _environmentVariableProvider.AwsSessionToken(false);
249-
250247
private string GetAzureStorageConnectionString(MigrateRepoCommandArgs args) => args.AzureStorageConnectionString.HasValue()
251248
? args.AzureStorageConnectionString
252249
: _environmentVariableProvider.AzureStorageConnectionString(false);
@@ -325,16 +322,9 @@ private void ValidateUploadOptions(MigrateRepoCommandArgs args)
325322
throw new OctoshiftCliException("Either --aws-secret-key or AWS_SECRET_ACCESS_KEY environment variable must be set.");
326323
}
327324

328-
if (GetAwsSessionToken(args).HasValue() && GetAwsRegion(args).IsNullOrWhiteSpace())
329-
{
330-
throw new OctoshiftCliException(
331-
"--aws-region or AWS_REGION environment variable must be provided with --aws-session-token or AWS_SESSION_TOKEN environment variable.");
332-
}
333-
334-
if (!GetAwsRegion(args).HasValue())
325+
if (GetAwsRegion(args).IsNullOrWhiteSpace())
335326
{
336-
_log.LogWarning("Specifying an AWS region with the --aws-region argument or AWS_REGION environment variable is currently not required, " +
337-
"but will be required in a future release. Defaulting to us-east-1.");
327+
throw new OctoshiftCliException("Either --aws-region or AWS_REGION environment variable must be set.");
338328
}
339329
}
340330
}

src/bbs2gh/Factories/AwsApiFactory.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ public virtual AwsApi Create(string awsRegion = null, string awsAccessKeyId = nu
1616
awsAccessKeyId ??= _environmentVariableProvider.AwsAccessKeyId();
1717
awsSecretAccessKey ??= _environmentVariableProvider.AwsSecretAccessKey();
1818
awsSessionToken ??= _environmentVariableProvider.AwsSessionToken(false);
19-
awsRegion ??= _environmentVariableProvider.AwsRegion(false);
19+
awsRegion ??= _environmentVariableProvider.AwsRegion();
2020

2121
return new AwsApi(awsAccessKeyId, awsSecretAccessKey, awsRegion, awsSessionToken);
2222
}

src/gei/Commands/GenerateScript/GenerateScriptCommand.cs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,7 @@ public GenerateScriptCommand() : base(
8585
public Option<string> AwsRegion { get; } = new("--aws-region")
8686
{
8787
Description = "If using AWS, the AWS region. If not provided, it will be read from AWS_REGION environment variable. " +
88-
"Defaults to us-east-1 if neither the argument nor the environment variable is set. " +
89-
"In a future release, you will be required to set an AWS region if using AWS S3 as your blob storage provider."
88+
"Required if using AWS."
9089
};
9190

9291
public Option<bool> Verbose { get; } = new("--verbose");

0 commit comments

Comments
 (0)