ibnezzoubayr/Atay-Makhzan
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feature non-zipped actions artifacts (action v7) (#36786)

Browse Source 
- content_encoding contains a slash => v4 artifact
- updated proto files to support mime_type and no longer return errors for upload-artifact v7
- json and txt files are now previewed in browser
- normalized content-disposition header creation
- azure blob storage uploads directly in servedirect mode (no proxying data)
- normalize content-disposition headers based on go mime package
  - getting both filename and filename* encoding is done via custom code

Closes #36829

-----

Signed-off-by: ChristopherHX <christopher.homberger@web.de>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
ChristopherHX
2026-03-25 17:37:48 +01:00
committed by GitHub
parent 435123fe65
commit bc5c554072
29 changed files with 1003 additions and 826 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration/api_actions_artifact_v4_test.go
+256 -94
View File
@@ -11,23 +11,28 @@ import (
"encoding/xml"
"fmt"
"io"
"mime"
"net/http"
"strings"
"testing"
"time"
actions_model "code.gitea.io/gitea/models/actions"
auth_model "code.gitea.io/gitea/models/auth"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/test"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/routers/api/actions"
actions_service "code.gitea.io/gitea/services/actions"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/types/known/timestamppb"
@@ -48,15 +53,18 @@ func TestActionsArtifactV4UploadSingleFile(t *testing.T) {
assert.NoError(t, err)
table := []struct {
name string
version int32
blockID bool
noLength bool
append int
name string
version int32
contentType string
blockID bool
noLength bool
append int
path string
}{
{
name: "artifact",
version: 4,
path: "artifact.zip",
},
{
name: "artifact2",
@@ -98,6 +106,23 @@ func TestActionsArtifactV4UploadSingleFile(t *testing.T) {
append: 4,
blockID: true,
},
{
name: "artifact9.json",
version: 7,
contentType: "application/json",
},
{
name: "artifact10",
version: 7,
contentType: "application/zip",
path: "artifact10.zip",
},
{
name: "artifact11.zip",
version: 7,
contentType: "application/zip",
path: "artifact11.zip",
},
}
for _, entry := range table {
@@ -108,6 +133,7 @@ func TestActionsArtifactV4UploadSingleFile(t *testing.T) {
Name: entry.name,
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
MimeType: util.Iif(entry.contentType != "", wrapperspb.String(entry.contentType), nil),
})).AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var uploadResp actions.CreateArtifactResponse
@@ -120,9 +146,8 @@ func TestActionsArtifactV4UploadSingleFile(t *testing.T) {
blocks := make([]string, 0, util.Iif(entry.blockID, entry.append+1, 0))
// get upload url
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
for i := range entry.append + 1 {
url := uploadResp.SignedUploadUrl[idx:]
url := uploadResp.SignedUploadUrl
// See https://learn.microsoft.com/en-us/rest/api/storageservices/append-block
// See https://learn.microsoft.com/en-us/rest/api/storageservices/put-block
if entry.blockID {
@@ -146,7 +171,7 @@ func TestActionsArtifactV4UploadSingleFile(t *testing.T) {
if entry.blockID && entry.append > 0 {
// https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-list
blockListURL := uploadResp.SignedUploadUrl[idx:] + "&comp=blocklist"
blockListURL := uploadResp.SignedUploadUrl + "&comp=blocklist"
// upload artifact blockList
blockList := &actions.BlockList{
Latest: blocks,
@@ -174,6 +199,19 @@ func TestActionsArtifactV4UploadSingleFile(t *testing.T) {
var finalizeResp actions.FinalizeArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)
assert.True(t, finalizeResp.Ok)
artifact := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionArtifact{ID: finalizeResp.ArtifactId})
if entry.contentType != "" {
assert.Equal(t, entry.contentType, artifact.ContentEncodingOrType)
} else {
assert.Equal(t, "application/zip", artifact.ContentEncodingOrType)
}
if entry.path != "" {
assert.Equal(t, entry.path, artifact.ArtifactPath)
}
assert.Equal(t, actions_model.ArtifactStatusUploadConfirmed, artifact.Status)
assert.Equal(t, int64(entry.append+1)*1024, artifact.FileSize)
assert.Equal(t, int64(entry.append+1)*1024, artifact.FileCompressedSize)
})
}
}
@@ -198,8 +236,7 @@ func TestActionsArtifactV4UploadSingleFileWrongChecksum(t *testing.T) {
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
// get upload url
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
url := uploadResp.SignedUploadUrl[idx:] + "&comp=block"
url := uploadResp.SignedUploadUrl + "&comp=block"
// upload artifact chunk
body := strings.Repeat("B", 1024)
@@ -243,8 +280,7 @@ func TestActionsArtifactV4UploadSingleFileWithRetentionDays(t *testing.T) {
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
// get upload url
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
url := uploadResp.SignedUploadUrl[idx:] + "&comp=block"
url := uploadResp.SignedUploadUrl + "&comp=block"
// upload artifact chunk
body := strings.Repeat("A", 1024)
@@ -290,9 +326,8 @@ func TestActionsArtifactV4UploadSingleFileWithPotentialHarmfulBlockID(t *testing
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
// get upload urls
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
url := uploadResp.SignedUploadUrl[idx:] + "&comp=block&blockid=%2f..%2fmyfile"
blockListURL := uploadResp.SignedUploadUrl[idx:] + "&comp=blocklist"
url := uploadResp.SignedUploadUrl + "&comp=block&blockid=%2f..%2fmyfile"
blockListURL := uploadResp.SignedUploadUrl + "&comp=blocklist"
// upload artifact chunk
body := strings.Repeat("A", 1024)
@@ -339,63 +374,126 @@ func TestActionsArtifactV4UploadSingleFileWithChunksOutOfOrder(t *testing.T) {
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
assert.NoError(t, err)
// acquire artifact upload url
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact", toProtoJSON(&actions.CreateArtifactRequest{
Version: 4,
Name: "artifactWithChunksOutOfOrder",
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var uploadResp actions.CreateArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &uploadResp)
assert.True(t, uploadResp.Ok)
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
// get upload urls
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
block1URL := uploadResp.SignedUploadUrl[idx:] + "&comp=block&blockid=block1"
block2URL := uploadResp.SignedUploadUrl[idx:] + "&comp=block&blockid=block2"
blockListURL := uploadResp.SignedUploadUrl[idx:] + "&comp=blocklist"
// upload artifact chunks
bodyb := strings.Repeat("B", 1024)
req = NewRequestWithBody(t, "PUT", block2URL, strings.NewReader(bodyb))
MakeRequest(t, req, http.StatusCreated)
bodya := strings.Repeat("A", 1024)
req = NewRequestWithBody(t, "PUT", block1URL, strings.NewReader(bodya))
MakeRequest(t, req, http.StatusCreated)
// upload artifact blockList
blockList := &actions.BlockList{
Latest: []string{
"block1",
"block2",
},
table := []struct {
name string
artifactName string
serveDirect bool
contentType string
}{
{name: "Upload-Zip", artifactName: "artifact-v4-upload", contentType: ""},
{name: "Upload-Pdf", artifactName: "report-upload.pdf", contentType: "application/pdf"},
{name: "Upload-Html", artifactName: "report-upload.html", contentType: "application/html"},
{name: "ServeDirect-Zip", artifactName: "artifact-v4-upload-serve-direct", contentType: "", serveDirect: true},
{name: "ServeDirect-Pdf", artifactName: "report-upload-serve-direct.pdf", contentType: "application/pdf", serveDirect: true},
{name: "ServeDirect-Html", artifactName: "report-upload-serve-direct.html", contentType: "application/html", serveDirect: true},
}
rawBlockList, err := xml.Marshal(blockList)
assert.NoError(t, err)
req = NewRequestWithBody(t, "PUT", blockListURL, bytes.NewReader(rawBlockList))
MakeRequest(t, req, http.StatusCreated)
t.Logf("Create artifact confirm")
for _, entry := range table {
t.Run(entry.name, func(t *testing.T) {
// Only AzureBlobStorageType supports ServeDirect Uploads
switch setting.Actions.ArtifactStorage.Type {
case setting.AzureBlobStorageType:
defer test.MockVariableValue(&setting.Actions.ArtifactStorage.AzureBlobConfig.ServeDirect, entry.serveDirect)()
default:
if entry.serveDirect {
t.Skip()
}
}
// acquire artifact upload url
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact", toProtoJSON(&actions.CreateArtifactRequest{
Version: util.Iif[int32](entry.contentType != "", 7, 4),
Name: entry.artifactName,
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
MimeType: util.Iif(entry.contentType != "", wrapperspb.String(entry.contentType), nil),
})).AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var uploadResp actions.CreateArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &uploadResp)
assert.True(t, uploadResp.Ok)
if !entry.serveDirect {
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
}
sha := sha256.Sum256([]byte(bodya + bodyb))
// get upload urls
block1URL := uploadResp.SignedUploadUrl + "&comp=block&blockid=" + base64.RawURLEncoding.EncodeToString([]byte("block1"))
block2URL := uploadResp.SignedUploadUrl + "&comp=block&blockid=" + base64.RawURLEncoding.EncodeToString([]byte("block2"))
blockListURL := uploadResp.SignedUploadUrl + "&comp=blocklist"
// confirm artifact upload
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/FinalizeArtifact", toProtoJSON(&actions.FinalizeArtifactRequest{
Name: "artifactWithChunksOutOfOrder",
Size: 2048,
Hash: wrapperspb.String("sha256:" + hex.EncodeToString(sha[:])),
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).
AddTokenAuth(token)
resp = MakeRequest(t, req, http.StatusOK)
var finalizeResp actions.FinalizeArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)
assert.True(t, finalizeResp.Ok)
// upload artifact chunks
bodyb := strings.Repeat("B", 1024)
req = NewRequestWithBody(t, "PUT", block2URL, strings.NewReader(bodyb))
if entry.serveDirect {
req.Request.RequestURI = ""
nresp, err := http.DefaultClient.Do(req.Request)
require.NoError(t, err)
nresp.Body.Close()
require.Equal(t, http.StatusCreated, nresp.StatusCode)
} else {
MakeRequest(t, req, http.StatusCreated)
}
bodya := strings.Repeat("A", 1024)
req = NewRequestWithBody(t, "PUT", block1URL, strings.NewReader(bodya))
if entry.serveDirect {
req.Request.RequestURI = ""
nresp, err := http.DefaultClient.Do(req.Request)
require.NoError(t, err)
nresp.Body.Close()
require.Equal(t, http.StatusCreated, nresp.StatusCode)
} else {
MakeRequest(t, req, http.StatusCreated)
}
// upload artifact blockList
blockList := &actions.BlockList{
Latest: []string{
base64.RawURLEncoding.EncodeToString([]byte("block1")),
base64.RawURLEncoding.EncodeToString([]byte("block2")),
},
}
rawBlockList, err := xml.Marshal(blockList)
assert.NoError(t, err)
req = NewRequestWithBody(t, "PUT", blockListURL, bytes.NewReader(rawBlockList))
if entry.serveDirect {
req.Request.RequestURI = ""
nresp, err := http.DefaultClient.Do(req.Request)
require.NoError(t, err)
nresp.Body.Close()
require.Equal(t, http.StatusCreated, nresp.StatusCode)
} else {
MakeRequest(t, req, http.StatusCreated)
}
t.Logf("Create artifact confirm")
sha := sha256.Sum256([]byte(bodya + bodyb))
// confirm artifact upload
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/FinalizeArtifact", toProtoJSON(&actions.FinalizeArtifactRequest{
Name: entry.artifactName,
Size: 2048,
Hash: wrapperspb.String("sha256:" + hex.EncodeToString(sha[:])),
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).
AddTokenAuth(token)
resp = MakeRequest(t, req, http.StatusOK)
var finalizeResp actions.FinalizeArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)
assert.True(t, finalizeResp.Ok)
artifact := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionArtifact{ID: finalizeResp.ArtifactId})
if entry.contentType != "" {
assert.Equal(t, entry.contentType, artifact.ContentEncodingOrType)
} else {
assert.Equal(t, "application/zip", artifact.ContentEncodingOrType)
}
assert.Equal(t, actions_model.ArtifactStatusUploadConfirmed, artifact.Status)
assert.Equal(t, int64(2048), artifact.FileSize)
assert.Equal(t, int64(2048), artifact.FileCompressedSize)
})
}
}
func TestActionsArtifactV4DownloadSingle(t *testing.T) {
@@ -404,33 +502,97 @@ func TestActionsArtifactV4DownloadSingle(t *testing.T) {
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
assert.NoError(t, err)
// list artifacts by name
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/ListArtifacts", toProtoJSON(&actions.ListArtifactsRequest{
NameFilter: wrapperspb.String("artifact-v4-download"),
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var listResp actions.ListArtifactsResponse
protojson.Unmarshal(resp.Body.Bytes(), &listResp)
assert.Len(t, listResp.Artifacts, 1)
table := []struct {
Name string
ArtifactName string
FileName string
ServeDirect bool
ContentType string
ContentDisposition string
}{
{Name: "Download-Zip", ArtifactName: "artifact-v4-download", FileName: "artifact-v4-download.zip", ContentType: "application/zip"},
{Name: "Download-Pdf", ArtifactName: "report.pdf", FileName: "report.pdf", ContentType: "application/pdf"},
{Name: "Download-Html", ArtifactName: "report.html", FileName: "report.html", ContentType: "application/html"},
{Name: "ServeDirect-Zip", ArtifactName: "artifact-v4-download", FileName: "artifact-v4-download.zip", ContentType: "application/zip", ServeDirect: true},
{Name: "ServeDirect-Pdf", ArtifactName: "report.pdf", FileName: "report.pdf", ContentType: "application/pdf", ServeDirect: true},
{Name: "ServeDirect-Html", ArtifactName: "report.html", FileName: "report.html", ContentType: "application/html", ServeDirect: true},
}
// acquire artifact download url
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/GetSignedArtifactURL", toProtoJSON(&actions.GetSignedArtifactURLRequest{
Name: "artifact-v4-download",
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).
AddTokenAuth(token)
resp = MakeRequest(t, req, http.StatusOK)
var finalizeResp actions.GetSignedArtifactURLResponse
protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)
assert.NotEmpty(t, finalizeResp.SignedUrl)
for _, entry := range table {
t.Run(entry.Name, func(t *testing.T) {
switch setting.Actions.ArtifactStorage.Type {
case setting.AzureBlobStorageType:
defer test.MockVariableValue(&setting.Actions.ArtifactStorage.AzureBlobConfig.ServeDirect, entry.ServeDirect)()
case setting.MinioStorageType:
defer test.MockVariableValue(&setting.Actions.ArtifactStorage.MinioConfig.ServeDirect, entry.ServeDirect)()
default:
if entry.ServeDirect {
t.Skip()
}
}
req = NewRequest(t, "GET", finalizeResp.SignedUrl)
resp = MakeRequest(t, req, http.StatusOK)
body := strings.Repeat("D", 1024)
assert.Equal(t, body, resp.Body.String())
// list artifacts by name
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/ListArtifacts", toProtoJSON(&actions.ListArtifactsRequest{
NameFilter: wrapperspb.String(entry.ArtifactName),
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var listResp actions.ListArtifactsResponse
require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &listResp))
require.Len(t, listResp.Artifacts, 1)
// list artifacts by id
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/ListArtifacts", toProtoJSON(&actions.ListArtifactsRequest{
IdFilter: wrapperspb.Int64(listResp.Artifacts[0].DatabaseId),
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).AddTokenAuth(token)
resp = MakeRequest(t, req, http.StatusOK)
require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &listResp))
assert.Len(t, listResp.Artifacts, 1)
// acquire artifact download url
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/GetSignedArtifactURL", toProtoJSON(&actions.GetSignedArtifactURLRequest{
Name: entry.ArtifactName,
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).
AddTokenAuth(token)
resp = MakeRequest(t, req, http.StatusOK)
var finalizeResp actions.GetSignedArtifactURLResponse
require.NoError(t, protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp))
assert.NotEmpty(t, finalizeResp.SignedUrl)
body := strings.Repeat("D", 1024)
var contentDisposition string
if entry.ServeDirect {
externalReq, err := http.NewRequestWithContext(t.Context(), http.MethodGet, finalizeResp.SignedUrl, nil)
require.NoError(t, err)
externalResp, err := http.DefaultClient.Do(externalReq)
require.NoError(t, err)
assert.Equal(t, http.StatusOK, externalResp.StatusCode)
assert.Equal(t, entry.ContentType, externalResp.Header.Get("Content-Type"))
contentDisposition = externalResp.Header.Get("Content-Disposition")
buf := make([]byte, 1024)
n, err := io.ReadAtLeast(externalResp.Body, buf, len(buf))
externalResp.Body.Close()
require.NoError(t, err)
assert.Equal(t, len(buf), n)
assert.Equal(t, body, string(buf))
} else {
req = NewRequest(t, "GET", finalizeResp.SignedUrl)
resp = MakeRequest(t, req, http.StatusOK)
assert.Equal(t, entry.ContentType, resp.Header().Get("Content-Type"))
contentDisposition = resp.Header().Get("Content-Disposition")
assert.Equal(t, body, resp.Body.String())
}
disposition, param, err := mime.ParseMediaType(contentDisposition)
require.NoError(t, err)
assert.Equal(t, "inline", disposition)
assert.Equal(t, entry.FileName, param["filename"])
})
}
}
func TestActionsArtifactV4RunDownloadSinglePublicApi(t *testing.T) {
@@ -561,7 +723,7 @@ func TestActionsArtifactV4ListAndGetPublicApi(t *testing.T) {
for _, artifact := range listResp.Entries {
assert.Contains(t, artifact.URL, fmt.Sprintf("/api/v1/repos/%s/actions/artifacts/%d", repo.FullName(), artifact.ID))
assert.Contains(t, artifact.ArchiveDownloadURL, fmt.Sprintf("/api/v1/repos/%s/actions/artifacts/%d/zip", repo.FullName(), artifact.ID))
req = NewRequestWithBody(t, "GET", listResp.Entries[0].URL, nil).
req = NewRequestWithBody(t, "GET", artifact.URL, nil).
AddTokenAuth(token)
resp = MakeRequest(t, req, http.StatusOK)