simon.suckut/Gitea
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix artifacts v4 backend upload problems (#36805)

Browse Source 
* Use base64.RawURLEncoding to avoid equal sign
  * using the nodejs package they seem to get lost
* Support uploads with unspecified length
* Support uploads with a single named blockid
  * without requiring a blockmap

---------

Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
ChristopherHX
2026-03-05 16:49:01 +01:00
committed by GitHub
parent 5d87bb3d45
commit 867c4af481
10 changed files with 552 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
tests/integration/api_actions_artifact_v4_test.go
View File

@@ -6,6 +6,7 @@ package integration
import (
"bytes"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/xml"
"fmt"
@@ -22,6 +23,7 @@ import (
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/storage"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/routers/api/actions"
actions_service "code.gitea.io/gitea/services/actions"
@@ -45,45 +47,135 @@ func TestActionsArtifactV4UploadSingleFile(t *testing.T) {
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
assert.NoError(t, err)
// acquire artifact upload url
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact", toProtoJSON(&actions.CreateArtifactRequest{
Version: 4,
Name: "artifact",
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var uploadResp actions.CreateArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &uploadResp)
assert.True(t, uploadResp.Ok)
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
table := []struct {
name string
version int32
blockID bool
noLength bool
append int
}{
{
name: "artifact",
version: 4,
},
{
name: "artifact2",
version: 4,
blockID: true,
},
{
name: "artifact3",
version: 4,
noLength: true,
},
{
name: "artifact4",
version: 4,
blockID: true,
noLength: true,
},
{
name: "artifact5",
version: 7,
blockID: true,
},
{
name: "artifact6",
version: 7,
append: 2,
noLength: true,
},
{
name: "artifact7",
version: 7,
append: 3,
blockID: true,
noLength: true,
},
{
name: "artifact8",
version: 7,
append: 4,
blockID: true,
},
}
// get upload url
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
url := uploadResp.SignedUploadUrl[idx:] + "&comp=block"
for _, entry := range table {
t.Run(entry.name, func(t *testing.T) {
// acquire artifact upload url
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact", toProtoJSON(&actions.CreateArtifactRequest{
Version: entry.version,
Name: entry.name,
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var uploadResp actions.CreateArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &uploadResp)
assert.True(t, uploadResp.Ok)
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
// upload artifact chunk
body := strings.Repeat("A", 1024)
req = NewRequestWithBody(t, "PUT", url, strings.NewReader(body))
MakeRequest(t, req, http.StatusCreated)
h := sha256.New()
t.Logf("Create artifact confirm")
blocks := make([]string, 0, util.Iif(entry.blockID, entry.append+1, 0))
sha := sha256.Sum256([]byte(body))
// get upload url
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
for i := range entry.append + 1 {
url := uploadResp.SignedUploadUrl[idx:]
// See https://learn.microsoft.com/en-us/rest/api/storageservices/append-block
// See https://learn.microsoft.com/en-us/rest/api/storageservices/put-block
if entry.blockID {
blockID := base64.RawURLEncoding.EncodeToString(fmt.Append([]byte("SOME_BIG_BLOCK_ID_"), i))
blocks = append(blocks, blockID)
url += "&comp=block&blockid=" + blockID
} else {
url += "&comp=appendBlock"
}
// confirm artifact upload
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/FinalizeArtifact", toProtoJSON(&actions.FinalizeArtifactRequest{
Name: "artifact",
Size: 1024,
Hash: wrapperspb.String("sha256:" + hex.EncodeToString(sha[:])),
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).
AddTokenAuth(token)
resp = MakeRequest(t, req, http.StatusOK)
var finalizeResp actions.FinalizeArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)
assert.True(t, finalizeResp.Ok)
// upload artifact chunk
body := strings.Repeat("A", 1024)
_, _ = h.Write([]byte(body))
var bodyReader io.Reader = strings.NewReader(body)
if entry.noLength {
bodyReader = io.MultiReader(bodyReader)
}
req = NewRequestWithBody(t, "PUT", url, bodyReader)
MakeRequest(t, req, http.StatusCreated)
}
if entry.blockID && entry.append > 0 {
// https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-list
blockListURL := uploadResp.SignedUploadUrl[idx:] + "&comp=blocklist"
// upload artifact blockList
blockList := &actions.BlockList{
Latest: blocks,
}
rawBlockList, err := xml.Marshal(blockList)
assert.NoError(t, err)
req = NewRequestWithBody(t, "PUT", blockListURL, bytes.NewReader(rawBlockList))
MakeRequest(t, req, http.StatusCreated)
}
sha := h.Sum(nil)
t.Logf("Create artifact confirm")
// confirm artifact upload
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/FinalizeArtifact", toProtoJSON(&actions.FinalizeArtifactRequest{
Name: entry.name,
Size: int64(entry.append+1) * 1024,
Hash: wrapperspb.String("sha256:" + hex.EncodeToString(sha)),
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).
AddTokenAuth(token)
resp = MakeRequest(t, req, http.StatusOK)
var finalizeResp actions.FinalizeArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)
assert.True(t, finalizeResp.Ok)
})
}
}
func TestActionsArtifactV4UploadSingleFileWrongChecksum(t *testing.T) {
@@ -312,7 +404,7 @@ func TestActionsArtifactV4DownloadSingle(t *testing.T) {
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
assert.NoError(t, err)
// acquire artifact upload url
// list artifacts by name
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/ListArtifacts", toProtoJSON(&actions.ListArtifactsRequest{
NameFilter: wrapperspb.String("artifact-v4-download"),
WorkflowRunBackendId: "792",
@@ -323,7 +415,7 @@ func TestActionsArtifactV4DownloadSingle(t *testing.T) {
protojson.Unmarshal(resp.Body.Bytes(), &listResp)
assert.Len(t, listResp.Artifacts, 1)
// confirm artifact upload
// acquire artifact download url
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/GetSignedArtifactURL", toProtoJSON(&actions.GetSignedArtifactURLRequest{
Name: "artifact-v4-download",
WorkflowRunBackendId: "792",

4
tests/integration/integration_test.go
View File

@@ -341,6 +341,10 @@ func MakeRequest(t testing.TB, rw *RequestWrapper, expectedStatus int) *httptest
if req.RemoteAddr == "" {
req.RemoteAddr = "test-mock:12345"
}
// Ensure unknown contentLength is seen as -1
if req.Body != nil && req.ContentLength == 0 {
req.ContentLength = -1
}
testWebRoutes.ServeHTTP(recorder, req)
if expectedStatus != NoExpectedStatus {
if expectedStatus != recorder.Code {