From 3ce9bbc7dcc8d66b935382e2f10e07d698c4e32e Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 25 Jun 2025 17:42:52 -0700 Subject: [PATCH] Consume URI in PUT response for GitHub-owned storage for GEI. --- src/Octoshift/Services/ArchiveUploader.cs | 16 ++--- .../Services/ArchiveUploadersTests.cs | 64 +++++++++++++++---- 2 files changed, 61 insertions(+), 19 deletions(-) diff --git a/src/Octoshift/Services/ArchiveUploader.cs b/src/Octoshift/Services/ArchiveUploader.cs index 78e2a3112..337074435 100644 --- a/src/Octoshift/Services/ArchiveUploader.cs +++ b/src/Octoshift/Services/ArchiveUploader.cs @@ -4,7 +4,6 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; -using System.Web; using Newtonsoft.Json.Linq; using OctoshiftCLI.Extensions; @@ -64,11 +63,8 @@ private async Task UploadMultipart(Stream archiveContent, string archive { // 1. Start the upload var startHeaders = await StartUpload(uploadUrl, archiveName, archiveContent.Length); - var nextUrl = GetNextUrl(startHeaders); - var guid = HttpUtility.ParseQueryString(nextUrl.Query)["guid"]; - // 2. Upload parts int bytesRead; var partsRead = 0; @@ -80,9 +76,9 @@ private async Task UploadMultipart(Stream archiveContent, string archive } // 3. Complete the upload - await CompleteUpload(nextUrl.ToString()); + var geiUri = await CompleteUpload(nextUrl.ToString()); - return $"gei://archive/{guid}"; + return geiUri.ToString(); } catch (Exception ex) { @@ -132,12 +128,16 @@ private async Task UploadPart(byte[] body, int bytesRead, string nextUrl, i } } - private async Task CompleteUpload(string lastUrl) + private async Task CompleteUpload(string lastUrl) { try { - await _retryPolicy.Retry(async () => await _client.PutAsync(lastUrl, "")); + var response = await _retryPolicy.Retry(async () => await _client.PutAsync(lastUrl, "")); + var responseData = JObject.Parse(response); + _log.LogInformation("Finished uploading archive"); + + return new Uri((string)responseData["uri"]); } catch (Exception ex) { diff --git a/src/OctoshiftCLI.Tests/Octoshift/Services/ArchiveUploadersTests.cs b/src/OctoshiftCLI.Tests/Octoshift/Services/ArchiveUploadersTests.cs index b47b302c9..f9bdff5d6 100644 --- a/src/OctoshiftCLI.Tests/Octoshift/Services/ArchiveUploadersTests.cs +++ b/src/OctoshiftCLI.Tests/Octoshift/Services/ArchiveUploadersTests.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using FluentAssertions; using Moq; +using Newtonsoft.Json.Linq; using OctoshiftCLI.Extensions; using OctoshiftCLI.Services; using Xunit; @@ -50,9 +51,20 @@ public async Task Upload_Should_Upload_All_Chunks_When_Stream_Exceeds_Limit() const string archiveName = "test-archive"; const string baseUrl = "https://uploads.github.com"; const string guid = "c9dbd27b-f190-4fe4-979f-d0b7c9b0fcb3"; + const string geiUri = $"gei://archive/{guid}"; var startUploadBody = new { content_type = "application/octet-stream", name = archiveName, size = contentSize }; + var completeUploadResponse = new JObject + { + ["guid"] = guid, + ["node_id"] = "global-relay-id", + ["name"] = archiveName, + ["size"] = largeContent.Length, + ["uri"] = geiUri, + ["created_at"] = "2025-06-23T17:13:02.818-07:00" + }; + const string initialUploadUrl = $"/organizations/{orgDatabaseId}/gei/archive/blobs/uploads"; const string firstUploadUrl = $"/organizations/{orgDatabaseId}/gei/archive/blobs/uploads?part_number=1&guid={guid}"; const string secondUploadUrl = $"/organizations/{orgDatabaseId}/gei/archive/blobs/uploads?part_number=2&guid={guid}"; @@ -81,13 +93,13 @@ public async Task Upload_Should_Upload_All_Chunks_When_Stream_Exceeds_Limit() // Mocking the final PUT request to complete the multipart upload _githubClientMock .Setup(m => m.PutAsync($"{baseUrl}{lastUrl}", "", null)) - .ReturnsAsync(string.Empty); + .ReturnsAsync(completeUploadResponse.ToString()); // act var result = await _archiveUploader.Upload(archiveContent, archiveName, orgDatabaseId); // assert - result.Should().Be($"gei://archive/{guid}"); + result.Should().Be(geiUri); _githubClientMock.Verify(m => m.PostWithFullResponseAsync(It.IsAny(), It.IsAny(), null), Times.Once); _githubClientMock.Verify(m => m.PatchWithFullResponseAsync(It.IsAny(), It.IsAny(), null), Times.Exactly(3)); @@ -107,10 +119,20 @@ public async Task Upload_Should_Retry_Failed_Upload_Part_Patch_Requests() const string archiveName = "test-archive"; const string baseUrl = "https://uploads.github.com"; const string guid = "c9dbd27b-f190-4fe4-979f-d0b7c9b0fcb3"; - const string expectedResult = $"gei://archive/{guid}"; + const string geiUri = $"gei://archive/{guid}"; var startUploadBody = new { content_type = "application/octet-stream", name = archiveName, size = largeContent.Length }; + var completeUploadResponse = new JObject + { + ["guid"] = guid, + ["node_id"] = "global-relay-id", + ["name"] = archiveName, + ["size"] = largeContent.Length, + ["uri"] = geiUri, + ["created_at"] = "2025-06-23T17:13:02.818-07:00" + }; + const string initialUploadUrl = $"/organizations/{orgDatabaseId}/gei/archive/blobs/uploads"; const string firstUploadUrl = $"/organizations/{orgDatabaseId}/gei/archive/blobs/uploads?part_number=1&guid={guid}"; const string secondUploadUrl = $"/organizations/{orgDatabaseId}/gei/archive/blobs/uploads?part_number=2&guid={guid}"; @@ -137,13 +159,13 @@ public async Task Upload_Should_Retry_Failed_Upload_Part_Patch_Requests() // Mocking the final PUT request to complete the multipart upload _githubClientMock .Setup(m => m.PutAsync($"{baseUrl}{lastUrl}", "", null)) - .ReturnsAsync(string.Empty); + .ReturnsAsync(completeUploadResponse.ToString()); // act var result = await _archiveUploader.Upload(archiveContent, archiveName, orgDatabaseId); // assert - Assert.Equal(expectedResult, result); + Assert.Equal(geiUri, result); _githubClientMock.Verify(m => m.PostWithFullResponseAsync(It.IsAny(), It.IsAny(), null), Times.Once); _githubClientMock.Verify(m => m.PatchWithFullResponseAsync(It.IsAny(), It.IsAny(), null), Times.Exactly(4)); // 2 retries + 2 success @@ -163,10 +185,20 @@ public async Task Upload_Should_Retry_Failed_Start_Upload_Post_Request() const string archiveName = "test-archive"; const string baseUrl = "https://uploads.github.com"; const string guid = "c9dbd27b-f190-4fe4-979f-d0b7c9b0fcb3"; - const string expectedResult = $"gei://archive/{guid}"; + const string geiUri = $"gei://archive/{guid}"; var startUploadBody = new { content_type = "application/octet-stream", name = archiveName, size = largeContent.Length }; + var completeUploadResponse = new JObject + { + ["guid"] = guid, + ["node_id"] = "global-relay-id", + ["name"] = archiveName, + ["size"] = largeContent.Length, + ["uri"] = geiUri, + ["created_at"] = "2025-06-23T17:13:02.818-07:00" + }; + const string initialUploadUrl = $"/organizations/{orgDatabaseId}/gei/archive/blobs/uploads"; const string firstUploadUrl = $"/organizations/{orgDatabaseId}/gei/archive/blobs/uploads?part_number=1&guid={guid}"; const string secondUploadUrl = $"/organizations/{orgDatabaseId}/gei/archive/blobs/uploads?part_number=2&guid={guid}"; @@ -193,13 +225,13 @@ public async Task Upload_Should_Retry_Failed_Start_Upload_Post_Request() // Mocking the final PUT request to complete the multipart upload _githubClientMock .Setup(m => m.PutAsync($"{baseUrl}{lastUrl}", "", null)) - .ReturnsAsync(string.Empty); + .ReturnsAsync(completeUploadResponse.ToString()); // act var result = await _archiveUploader.Upload(archiveContent, archiveName, orgDatabaseId); // assert - Assert.Equal(expectedResult, result); + Assert.Equal(geiUri, result); _githubClientMock.Verify(m => m.PostWithFullResponseAsync(It.IsAny(), It.IsAny(), null), Times.Exactly(3)); // 2 retries + 1 success _githubClientMock.Verify(m => m.PatchWithFullResponseAsync(It.IsAny(), It.IsAny(), null), Times.Exactly(2)); @@ -219,10 +251,20 @@ public async Task Upload_Should_Retry_Failed_Complete_Upload_Put_Request() const string archiveName = "test-archive"; const string baseUrl = "https://uploads.github.com"; const string guid = "c9dbd27b-f190-4fe4-979f-d0b7c9b0fcb3"; - const string expectedResult = $"gei://archive/{guid}"; + const string geiUri = $"gei://archive/{guid}"; var startUploadBody = new { content_type = "application/octet-stream", name = archiveName, size = largeContent.Length }; + var completeUploadResponse = new JObject + { + ["guid"] = guid, + ["node_id"] = "global-relay-id", + ["name"] = archiveName, + ["size"] = largeContent.Length, + ["uri"] = geiUri, + ["created_at"] = "2025-06-23T17:13:02.818-07:00" + }; + const string initialUploadUrl = $"/organizations/{orgDatabaseId}/gei/archive/blobs/uploads"; const string firstUploadUrl = $"/organizations/{orgDatabaseId}/gei/archive/blobs/uploads?part_number=1&guid={guid}"; const string secondUploadUrl = $"/organizations/{orgDatabaseId}/gei/archive/blobs/uploads?part_number=2&guid={guid}"; @@ -249,13 +291,13 @@ public async Task Upload_Should_Retry_Failed_Complete_Upload_Put_Request() .SetupSequence(m => m.PutAsync($"{baseUrl}{lastUrl}", "", null)) .ThrowsAsync(new TimeoutException("The operation was canceled.")) .ThrowsAsync(new TimeoutException("The operation was canceled.")) - .ReturnsAsync(string.Empty); + .ReturnsAsync(completeUploadResponse.ToString()); // act var result = await _archiveUploader.Upload(archiveContent, archiveName, orgDatabaseId); // assert - Assert.Equal(expectedResult, result); + Assert.Equal(geiUri, result); _githubClientMock.Verify(m => m.PostWithFullResponseAsync(It.IsAny(), It.IsAny(), null), Times.Once); _githubClientMock.Verify(m => m.PatchWithFullResponseAsync(It.IsAny(), It.IsAny(), null), Times.Exactly(2));