diff --git a/.gitignore b/.gitignore index 7b8c36c..6f46bb5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea -ocisrv \ No newline at end of file +ocisrv +/conformance/results/ diff --git a/Taskfile.yml b/Taskfile.yml index 5014961..e0a4580 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -45,3 +45,15 @@ tasks: cmds: - task: lint - task: test + + conformance: + desc: Run OCI distribution conformance tests against all supported backends + cmd: go test -tags=integration ./conformance -run TestOCIConformance -count=1 -v + + conformance:ocimem: + desc: Run OCI distribution conformance tests against ocimem + cmd: go test -tags=integration ./conformance -run TestOCIConformance/ocimem -count=1 -v + + conformance:ocilayout: + desc: Run OCI distribution conformance tests against ocilayout + cmd: go test -tags=integration ./conformance -run TestOCIConformance/ocilayout -count=1 -v diff --git a/cmd/ocisrv/go.mod b/cmd/ocisrv/go.mod index 3a0ba57..317d85b 100644 --- a/cmd/ocisrv/go.mod +++ b/cmd/ocisrv/go.mod @@ -4,9 +4,8 @@ go 1.25.0 require ( github.com/cue-exp/cueconfig v0.0.1 - github.com/go-json-experiment/json v0.0.0-20240524174822-2d9f40f7385b github.com/docker/oci v0.0.0 - github.com/opencontainers/go-digest v1.0.0 + github.com/go-json-experiment/json v0.0.0-20240524174822-2d9f40f7385b github.com/rogpeppe/go-internal v1.14.1 github.com/rogpeppe/retry v0.1.0 ) @@ -21,7 +20,6 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0 // indirect golang.org/x/mod v0.31.0 // indirect golang.org/x/net v0.48.0 // indirect diff --git a/cmd/ocisrv/go.sum b/cmd/ocisrv/go.sum index 01445f7..1eda9d2 100644 --- a/cmd/ocisrv/go.sum +++ b/cmd/ocisrv/go.sum @@ -30,10 +30,6 @@ github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQ github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9ZPiLVHXz3UFw2+psEX+gYcto= github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuHZgRjZUWWuH1DTxCtxbHDOIJsudS8jzY= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= -github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0 h1:sadMIsgmHpEOGbUs6VtHBXRR1OHevnj7hLx9ZcdNGW4= diff --git a/cmd/ocisrv/main.go b/cmd/ocisrv/main.go index 2bdf80c..1791a98 100644 --- a/cmd/ocisrv/main.go +++ b/cmd/ocisrv/main.go @@ -85,7 +85,11 @@ func main1() error { writeNetAddr(l) } fmt.Printf("listening on %v\n", l.Addr()) - err = http.Serve(l, ociserver.New(r, nil)) + srv, err := ociserver.New(r, nil) + if err != nil { + return fmt.Errorf("cannot construct server: %v", err) + } + err = http.Serve(l, srv) return fmt.Errorf("http server error: %v", err) } diff --git a/cmd/ocisrv/main_test.go b/cmd/ocisrv/main_test.go index 3a2eeb7..f3e90cf 100644 --- a/cmd/ocisrv/main_test.go +++ b/cmd/ocisrv/main_test.go @@ -27,7 +27,9 @@ import ( "testing" "time" + "github.com/docker/oci" "github.com/docker/oci/ociclient" + digest "github.com/docker/oci/ocidigest" "github.com/rogpeppe/go-internal/testscript" "github.com/rogpeppe/retry" ) diff --git a/conformance/Dockerfile b/conformance/Dockerfile new file mode 100644 index 0000000..7c7ba8e --- /dev/null +++ b/conformance/Dockerfile @@ -0,0 +1,21 @@ +FROM golang:1.25-alpine AS build + +ARG DISTRIBUTION_SPEC_REF=967efdc079b91785ad18c77cc4f8991a47feefbf + +RUN apk add --no-cache ca-certificates git +RUN mkdir /src \ + && cd /src \ + && git init \ + && git remote add origin https://github.com/opencontainers/distribution-spec.git \ + && git fetch --depth 1 origin "${DISTRIBUTION_SPEC_REF}" \ + && git checkout --detach FETCH_HEAD + +WORKDIR /src/conformance +RUN CGO_ENABLED=0 go build -o /conformance . + +FROM alpine:3.21 + +RUN apk add --no-cache ca-certificates +COPY --from=build /conformance /conformance +WORKDIR /work +ENTRYPOINT ["/conformance"] diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..6fc480c --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,18 @@ +# OCI distribution conformance + +This directory runs the official OCI distribution-spec conformance suite +against `ociserver` backed by the registry implementations in this repository. + +Docker must be installed and running. Run every backend with: + +```console +task conformance +``` + +Individual backends can be selected with `task conformance:ocimem` or +`task conformance:ocilayout`. + +The harness builds a pinned version of the upstream conformance runner, starts +each registry on a temporary local port, and fails when the upstream runner +reports a conformance failure. HTML, YAML, and JUnit reports are written under +`conformance/results//`. diff --git a/conformance/conformance_test.go b/conformance/conformance_test.go new file mode 100644 index 0000000..f9b8919 --- /dev/null +++ b/conformance/conformance_test.go @@ -0,0 +1,203 @@ +//go:build integration + +package conformance + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "os" + "os/exec" + "os/user" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/docker/oci" + "github.com/docker/oci/ocilayout" + "github.com/docker/oci/ocimem" + "github.com/docker/oci/ociserver" +) + +const conformanceImage = "docker-oci-conformance:integration" + +type backendCase struct { + name string + new func(*testing.T) oci.Interface +} + +func TestOCIConformance(t *testing.T) { + if testing.Short() { + t.Skip("skipping OCI conformance tests in short mode") + } + + root := repositoryRoot(t) + requireDocker(t, root) + buildConformanceImage(t, root) + + backends := []backendCase{ + { + name: "ocimem", + new: func(*testing.T) oci.Interface { + return ocimem.New() + }, + }, + { + name: "ocilayout", + new: func(t *testing.T) oci.Interface { + r, err := ocilayout.New(t.TempDir(), nil) + if err != nil { + t.Fatalf("creating ocilayout backend: %v", err) + } + return r + }, + }, + } + + for _, backend := range backends { + t.Run(backend.name, func(t *testing.T) { + runConformance(t, root, backend.name, backend.new(t)) + }) + } +} + +func runConformance(t *testing.T, root, backendName string, backend oci.Interface) { + t.Helper() + + handler, err := ociserver.New(backend, nil) + if err != nil { + t.Fatalf("creating OCI server: %v", err) + } + listener, err := net.Listen("tcp4", "0.0.0.0:0") + if err != nil { + t.Fatalf("listening for OCI server: %v", err) + } + httpServer := &http.Server{ + Handler: handler, + ReadHeaderTimeout: 30 * time.Second, + IdleTimeout: 120 * time.Second, + } + serveErr := make(chan error, 1) + go func() { + serveErr <- httpServer.Serve(listener) + }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := httpServer.Shutdown(ctx); err != nil { + t.Errorf("shutting down OCI server: %v", err) + _ = httpServer.Close() + } + select { + case err := <-serveErr: + if err != nil && !errors.Is(err, http.ErrServerClosed) { + t.Errorf("serving OCI registry: %v", err) + } + case <-time.After(time.Second): + t.Error("timed out waiting for OCI server to stop") + } + }) + + port := listener.Addr().(*net.TCPAddr).Port + waitForRegistry(t, fmt.Sprintf("http://127.0.0.1:%d/v2/", port)) + + resultsDir := filepath.Join(root, "conformance", "results", backendName) + if err := os.RemoveAll(resultsDir); err != nil { + t.Fatalf("removing old conformance results: %v", err) + } + if err := os.MkdirAll(resultsDir, 0o755); err != nil { + t.Fatalf("creating conformance results directory: %v", err) + } + + runID := strconv.FormatInt(time.Now().UnixNano(), 36) + args := []string{ + "run", "--rm", + "--add-host=host.docker.internal:host-gateway", + "-v", filepath.Join(root, "conformance", "oci-conformance.yaml") + ":/work/oci-conformance.yaml:ro", + "-v", resultsDir + ":/results", + "-e", fmt.Sprintf("OCI_REGISTRY=host.docker.internal:%d", port), + "-e", "OCI_REPO1=conformance/" + backendName + "/" + runID + "/repo1", + "-e", "OCI_REPO2=conformance/" + backendName + "/" + runID + "/repo2", + } + if currentUser, err := user.Current(); err == nil && currentUser.Uid != "" && currentUser.Gid != "" { + args = append(args, "--user", currentUser.Uid+":"+currentUser.Gid) + } + args = append(args, conformanceImage) + + runCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + output, err := runCommand(runCtx, root, "docker", args...) + t.Logf("OCI conformance output for %s:\n%s", backendName, output) + if err != nil { + t.Fatalf("OCI conformance failed for %s: %v; reports: %s", backendName, err, resultsDir) + } +} + +func repositoryRoot(t *testing.T) string { + t.Helper() + wd, err := os.Getwd() + if err != nil { + t.Fatalf("getting working directory: %v", err) + } + root, err := filepath.Abs(filepath.Join(wd, "..")) + if err != nil { + t.Fatalf("resolving repository root: %v", err) + } + return root +} + +func requireDocker(t *testing.T, root string) { + t.Helper() + if output, err := runCommand(context.Background(), root, "docker", "version"); err != nil { + t.Fatalf("Docker is required for conformance tests: %v\n%s", err, output) + } +} + +func buildConformanceImage(t *testing.T, root string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + output, err := runCommand(ctx, root, "docker", "build", + "-f", "conformance/Dockerfile", + "-t", conformanceImage, + "conformance/", + ) + if err != nil { + t.Fatalf("building OCI conformance image: %v\n%s", err, output) + } +} + +func waitForRegistry(t *testing.T, registryURL string) { + t.Helper() + client := &http.Client{Timeout: time.Second} + deadline := time.Now().Add(30 * time.Second) + var lastErr error + for time.Now().Before(deadline) { + resp, err := client.Get(registryURL) + if err == nil { + _ = resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return + } + lastErr = fmt.Errorf("status %s", resp.Status) + } else { + lastErr = err + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("timed out waiting for registry %s: %v", registryURL, lastErr) +} + +func runCommand(ctx context.Context, dir, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + output, err := cmd.CombinedOutput() + if err != nil { + return string(output), fmt.Errorf("%s %s: %w", name, strings.Join(args, " "), err) + } + return string(output), nil +} diff --git a/conformance/oci-conformance.yaml b/conformance/oci-conformance.yaml new file mode 100644 index 0000000..5d63698 --- /dev/null +++ b/conformance/oci-conformance.yaml @@ -0,0 +1,50 @@ +tls: disabled +resultsDir: /results +version: "1.1" +logging: warn + +# The integration harness overrides these for every backend and run. +registry: host.docker.internal:5000 +repo1: conformance/repo1 +repo2: conformance/repo2 + +username: "" +password: "" + +apis: + ping: true + pull: true + push: true + blobs: + atomic: true + delete: true + digestHeader: true + mountAnonymous: true + uploadCancel: true + manifests: + atomic: true + delete: true + digestHeader: true + tagParam: true + tags: + atomic: true + delete: true + list: true + referrer: true + +data: + image: true + index: true + indexList: true + sparse: false + artifact: true + subject: true + subjectMissing: true + artifactList: true + subjectList: true + dataField: true + nondistributable: true + customFields: true + noLayers: true + emptyBlob: true + sha512: true diff --git a/error.go b/error.go index a686a10..8dd75ae 100644 --- a/error.go +++ b/error.go @@ -332,6 +332,7 @@ var ( ErrDenied = NewError("requested access to the resource is denied", "DENIED", nil) ErrUnsupported = NewError("the operation is unsupported", "UNSUPPORTED", nil) ErrTooManyRequests = NewError("too many requests", "TOOMANYREQUESTS", nil) + ErrReferenced = NewError("referenced by another object", "DENIED", nil) // ErrRangeInvalid allows Interface implementations to reject invalid ranges, // such as a chunked upload PATCH not following the range from a previous PATCH. diff --git a/internal/ocirequest/create.go b/internal/ocirequest/create.go deleted file mode 100644 index 476a4fc..0000000 --- a/internal/ocirequest/create.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2023 CUE Labs AG -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ocirequest - -import ( - "encoding/base64" - "fmt" - "net/url" -) - -func (req *Request) Construct() (method string, ustr string, err error) { - method, ustr = req.construct() - u, err := url.Parse(ustr) - if err != nil { - return "", "", fmt.Errorf("invalid OCI request: %v", err) - } - if _, err := Parse(method, u); err != nil { - return "", "", fmt.Errorf("invalid OCI request: %v", err) - } - return method, ustr, nil -} - -func (req *Request) MustConstruct() (method string, ustr string) { - method, ustr, err := req.Construct() - if err != nil { - panic(err) - } - return method, ustr -} - -func (req *Request) construct() (method string, urlStr string) { - switch req.Kind { - case ReqPing: - return "GET", "/v2/" - case ReqBlobGet: - return "GET", "/v2/" + req.Repo + "/blobs/" + req.Digest - case ReqBlobHead: - return "HEAD", "/v2/" + req.Repo + "/blobs/" + req.Digest - case ReqBlobDelete: - return "DELETE", "/v2/" + req.Repo + "/blobs/" + req.Digest - case ReqBlobStartUpload: - return "POST", "/v2/" + req.Repo + "/blobs/uploads/" - case ReqBlobUploadBlob: - return "POST", "/v2/" + req.Repo + "/blobs/uploads/?digest=" + req.Digest - case ReqBlobMount: - return "POST", "/v2/" + req.Repo + "/blobs/uploads/?mount=" + req.Digest + "&from=" + req.FromRepo - case ReqBlobUploadInfo: - // Note: this is specific to the ociserver implementation. - return "GET", req.uploadPath() - case ReqBlobUploadChunk: - // Note: this is specific to the ociserver implementation. - return "PATCH", req.uploadPath() - case ReqBlobCompleteUpload: - // Note: this is specific to the ociserver implementation. - // TODO this is bogus when the upload ID contains query parameters. - return "PUT", req.uploadPath() + "?digest=" + req.Digest - case ReqManifestGet: - return "GET", "/v2/" + req.Repo + "/manifests/" + req.tagOrDigest() - case ReqManifestHead: - return "HEAD", "/v2/" + req.Repo + "/manifests/" + req.tagOrDigest() - case ReqManifestPut: - u := "/v2/" + req.Repo + "/manifests/" + req.tagOrDigest() - if len(req.Tags) > 0 { - params := url.Values{} - for _, tag := range req.Tags { - params.Add("tag", tag) - } - u += "?" + params.Encode() - } - return "PUT", u - case ReqManifestDelete: - return "DELETE", "/v2/" + req.Repo + "/manifests/" + req.tagOrDigest() - case ReqTagsList: - return "GET", "/v2/" + req.Repo + "/tags/list" + req.listParams() - case ReqReferrersList: - p := "/v2/" + req.Repo + "/referrers/" + req.Digest - if req.ArtifactType != "" { - p += "?" + url.Values{"artifactType": {req.ArtifactType}}.Encode() - } - return "GET", p - case ReqCatalogList: - return "GET", "/v2/_catalog" + req.listParams() - default: - panic("invalid request kind") - } -} - -func (req *Request) uploadPath() string { - return "/v2/" + req.Repo + "/blobs/uploads/" + base64.RawURLEncoding.EncodeToString([]byte(req.UploadID)) -} - -func (req *Request) listParams() string { - q := make(url.Values) - if req.ListN >= 0 { - q.Set("n", fmt.Sprint(req.ListN)) - } - if req.ListLast != "" { - q.Set("last", req.ListLast) - } - if len(q) > 0 { - return "?" + q.Encode() - } - return "" -} - -func (req *Request) tagOrDigest() string { - if req.Tag != "" { - return req.Tag - } - return req.Digest -} diff --git a/internal/ocirequest/request.go b/internal/ocirequest/request.go deleted file mode 100644 index ac5c41d..0000000 --- a/internal/ocirequest/request.go +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright 2023 CUE Labs AG -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ocirequest - -import ( - "encoding/base64" - "fmt" - "net/http" - "net/url" - "strconv" - "strings" - "unicode/utf8" - - "github.com/docker/oci" - "github.com/docker/oci/ociref" -) - -var ( - errBadlyFormedDigest = oci.NewError("badly formed digest", oci.ErrDigestInvalid.Code(), nil) - errMethodNotAllowed = httpErrorf(http.StatusMethodNotAllowed, "method not allowed") - errNotFound = httpErrorf(http.StatusNotFound, "page not found") -) - -func badRequestf(f string, a ...any) error { - return httpErrorf(http.StatusBadRequest, f, a...) -} - -func httpErrorf(statusCode int, f string, a ...any) error { - return oci.NewHTTPError(fmt.Errorf(f, a...), statusCode, nil, nil) -} - -type Request struct { - Kind Kind - - // Repo holds the repository name. Valid for all request kinds - // except ReqCatalogList and ReqPing. - Repo string - - // Digest holds the digest being used in the request. - // Valid for: - // ReqBlobMount - // ReqBlobUploadBlob - // ReqBlobGet - // ReqBlobHead - // ReqBlobDelete - // ReqBlobCompleteUpload - // ReqReferrersList - // - // Valid for these manifest requests when they're referring to a digest - // rather than a tag: - // ReqManifestGet - // ReqManifestHead - // ReqManifestPut - // ReqManifestDelete - Digest string - - // Tag holds the tag being used in the request. Valid for - // these manifest requests when they're referring to a tag: - // ReqManifestGet - // ReqManifestHead - // ReqManifestPut - // ReqManifestDelete - Tag string - - // Tags hold a list of tags to push with a manifest digest. Valid - // for these manifest requests: - // ReqManifestPut - Tags []string - - // FromRepo holds the repository name to mount from - // for ReqBlobMount. - FromRepo string - - // UploadID holds the upload identifier as used for - // chunked uploads. - // Valid for: - // ReqBlobUploadInfo - // ReqBlobUploadChunk - UploadID string - - // ListN holds the maximum count for listing. - // It's -1 to specify that all items should be returned. - // - // Valid for: - // ReqTagsList - // ReqCatalog - ListN int - - // ListLast holds the item to start just after - // when listing. - // - // Valid for: - // ReqTagsList - // ReqCatalog - ListLast string - - // ArtifactType holds the artifact type to filter by when - // listing. - // - // Valid for: - // ReqReferrersList - ArtifactType string -} - -type Kind int - -const ( - // end-1 GET /v2/ 200 404/401 - ReqPing = Kind(iota) - - // Blob-related endpoints - - // end-2 GET /v2//blobs/ 200 404 - ReqBlobGet - - // end-2 HEAD /v2//blobs/ 200 404 - ReqBlobHead - - // end-10 DELETE /v2//blobs/ 202 404/405 - ReqBlobDelete - - // end-4a POST /v2//blobs/uploads/ 202 404 - ReqBlobStartUpload - - // end-4b POST /v2//blobs/uploads/?digest= 201/202 404/400 - ReqBlobUploadBlob - - // end-11 POST /v2//blobs/uploads/?mount=&from= 201 404 - ReqBlobMount - - // end-13 GET /v2//blobs/uploads/ 204 404 - // NOTE: despite being described in the distribution spec, this - // isn't really part of the OCI spec. - ReqBlobUploadInfo - - // end-5 PATCH /v2//blobs/uploads/ 202 404/416 - // NOTE: despite being described in the distribution spec, this - // isn't really part of the OCI spec. - ReqBlobUploadChunk - - // end-6 PUT /v2//blobs/uploads/?digest= 201 404/400 - // NOTE: despite being described in the distribution spec, this - // isn't really part of the OCI spec. - ReqBlobCompleteUpload - - // Manifest-related endpoints - - // end-3 GET /v2//manifests/ 200 404 - ReqManifestGet - - // end-3 HEAD /v2//manifests/ 200 404 - ReqManifestHead - - // end-7 PUT /v2//manifests/ 201 404 - ReqManifestPut - - // end-9 DELETE /v2//manifests/ 202 404/400/405 - ReqManifestDelete - - // Tag-related endpoints - - // end-8a GET /v2//tags/list 200 404 - // end-8b GET /v2//tags/list?n=&last= 200 404 - ReqTagsList - - // Referrer-related endpoints - - // end-12a GET /v2//referrers/ 200 404/400 - ReqReferrersList - - // Catalog endpoints (out-of-spec) - // GET /v2/_catalog - ReqCatalogList -) - -// Parse parses the given HTTP method and URL as an OCI registry request. -// It understands the endpoints described in the [distribution spec]. -// -// If it returns an error, it will be of type [oci.Error] or [oci.HTTPError]. -// -// [distribution spec]: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#endpoints -func Parse(method string, u *url.URL) (*Request, error) { - path := u.Path - urlq, err := url.ParseQuery(u.RawQuery) - if err != nil { - return nil, badRequestf("invalid query parameters: %v", err) - } - - var rreq Request - if path == "/v2" || path == "/v2/" { - rreq.Kind = ReqPing - return &rreq, nil - } - path, ok := strings.CutPrefix(path, "/v2/") - if !ok { - return nil, oci.NewError("unknown URL path", oci.ErrNameUnknown.Code(), nil) - } - if path == "_catalog" { - if method != "GET" { - return nil, errMethodNotAllowed - } - rreq.Kind = ReqCatalogList - setListQueryParams(&rreq, urlq) - return &rreq, nil - } - uploadPath, ok := strings.CutSuffix(path, "/blobs/uploads/") - if !ok { - uploadPath, ok = strings.CutSuffix(path, "/blobs/uploads") - } - if ok { - rreq.Repo = uploadPath - if !ociref.IsValidRepository(rreq.Repo) { - return nil, oci.ErrNameInvalid - } - if method != "POST" { - return nil, errMethodNotAllowed - } - if d := urlq.Get("mount"); d != "" { - // end-11 - rreq.Digest = d - if !ociref.IsValidDigest(rreq.Digest) { - return nil, oci.ErrDigestInvalid - } - rreq.FromRepo = urlq.Get("from") - if rreq.FromRepo == "" { - // There's no "from" argument so fall back to - // a regular chunked upload. - rreq.Kind = ReqBlobStartUpload - // TODO does the "mount" query argument actually take effect in some way? - rreq.Digest = "" - return &rreq, nil - } - if !ociref.IsValidRepository(rreq.FromRepo) { - return nil, oci.ErrNameInvalid - } - rreq.Kind = ReqBlobMount - return &rreq, nil - } - if d := urlq.Get("digest"); d != "" { - // end-4b - rreq.Digest = d - if !ociref.IsValidDigest(d) { - return nil, errBadlyFormedDigest - } - rreq.Kind = ReqBlobUploadBlob - return &rreq, nil - } - // end-4a - rreq.Kind = ReqBlobStartUpload - return &rreq, nil - } - path, last, ok := cutLast(path, "/") - if !ok { - return nil, errNotFound - } - path, lastButOne, ok := cutLast(path, "/") - if !ok { - return nil, errNotFound - } - switch lastButOne { - case "blobs": - rreq.Repo = path - if !ociref.IsValidDigest(last) { - return nil, errBadlyFormedDigest - } - if !ociref.IsValidRepository(rreq.Repo) { - return nil, oci.ErrNameInvalid - } - rreq.Digest = last - switch method { - case "GET": - rreq.Kind = ReqBlobGet - case "HEAD": - rreq.Kind = ReqBlobHead - case "DELETE": - rreq.Kind = ReqBlobDelete - default: - return nil, errMethodNotAllowed - } - return &rreq, nil - case "uploads": - // Note: this section is all specific to ociserver and - // isn't part of the OCI registry spec. - repo, ok := strings.CutSuffix(path, "/blobs") - if !ok { - return nil, errNotFound - } - rreq.Repo = repo - if !ociref.IsValidRepository(rreq.Repo) { - return nil, oci.ErrNameInvalid - } - uploadID64 := last - if uploadID64 == "" { - return nil, errNotFound - } - uploadID, err := base64.RawURLEncoding.DecodeString(uploadID64) - if err != nil { - return nil, badRequestf("invalid upload ID %q (cannot decode)", uploadID64) - } - if !utf8.Valid(uploadID) { - return nil, badRequestf("upload ID %q decoded to invalid utf8", uploadID64) - } - rreq.UploadID = string(uploadID) - - switch method { - case "GET": - rreq.Kind = ReqBlobUploadInfo - case "PATCH": - rreq.Kind = ReqBlobUploadChunk - case "PUT": - rreq.Kind = ReqBlobCompleteUpload - rreq.Digest = urlq.Get("digest") - if !ociref.IsValidDigest(rreq.Digest) { - return nil, errBadlyFormedDigest - } - default: - return nil, errMethodNotAllowed - } - return &rreq, nil - case "manifests": - rreq.Repo = path - if !ociref.IsValidRepository(rreq.Repo) { - return nil, oci.ErrNameInvalid - } - switch { - case ociref.IsValidDigest(last): - rreq.Digest = last - case ociref.IsValidTag(last): - rreq.Tag = last - default: - return nil, errNotFound - } - switch method { - case "GET": - rreq.Kind = ReqManifestGet - case "HEAD": - rreq.Kind = ReqManifestHead - case "PUT": - rreq.Kind = ReqManifestPut - rreq.Tags = urlq["tag"] - case "DELETE": - rreq.Kind = ReqManifestDelete - default: - return nil, errMethodNotAllowed - } - return &rreq, nil - - case "tags": - if last != "list" { - return nil, errNotFound - } - if err := setListQueryParams(&rreq, urlq); err != nil { - return nil, err - } - if method != "GET" { - return nil, errMethodNotAllowed - } - rreq.Repo = path - if !ociref.IsValidRepository(rreq.Repo) { - return nil, oci.ErrNameInvalid - } - rreq.Kind = ReqTagsList - return &rreq, nil - case "referrers": - if !ociref.IsValidDigest(last) { - return nil, errBadlyFormedDigest - } - if method != "GET" { - return nil, errMethodNotAllowed - } - rreq.Repo = path - if !ociref.IsValidRepository(rreq.Repo) { - return nil, oci.ErrNameInvalid - } - // Unlike other list-oriented endpoints, there appears to be no defined way for the client - // to indicate the desired number of results, but set ListN anyway to be future-proof. - rreq.ListN = -1 - rreq.Digest = last - rreq.ArtifactType = urlq.Get("artifactType") - rreq.Kind = ReqReferrersList - return &rreq, nil - } - return nil, errNotFound -} - -func setListQueryParams(rreq *Request, urlq url.Values) error { - rreq.ListN = -1 - if nstr := urlq.Get("n"); nstr != "" { - n, err := strconv.Atoi(nstr) - if err != nil { - return badRequestf("query parameter n is not a valid integer") - } - rreq.ListN = n - } - rreq.ListLast = urlq.Get("last") - return nil -} - -func cutLast(s, sep string) (before, after string, found bool) { - if i := strings.LastIndex(s, sep); i >= 0 { - return s[:i], s[i+len(sep):], true - } - return "", s, false -} - -// ParseRange extracts the start and end offsets from a Content-Range string. -// The resulting start is inclusive and the end exclusive, to match Go convention, -// whereas Content-Range is inclusive on both ends. -func ParseRange(s string) (start, end int64, ok bool) { - p0s, p1s, ok := strings.Cut(s, "-") - if !ok { - return 0, 0, false - } - p0, err0 := strconv.ParseInt(p0s, 10, 64) - p1, err1 := strconv.ParseInt(p1s, 10, 64) - if p1 > 0 { - p1++ - } - return p0, p1, err0 == nil && err1 == nil -} - -// RangeString formats a pair of start and end offsets in the Content-Range form. -// The input start is inclusive and the end exclusive, to match Go convention, -// whereas Content-Range is inclusive on both ends. -func RangeString(start, end int64) string { - end-- - if end < 0 { - end = 0 - } - return fmt.Sprintf("%d-%d", start, end) -} diff --git a/internal/ocirequest/request_test.go b/internal/ocirequest/request_test.go deleted file mode 100644 index 3f9e5a0..0000000 --- a/internal/ocirequest/request_test.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2023 CUE Labs AG -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ocirequest - -import ( - "net/url" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -var parseRequestTests = []struct { - testName string - method string - url string - - wantRequest *Request - wantError string - wantConstruct string -}{{ - testName: "ping", - method: "GET", - url: "/v2", - wantRequest: &Request{ - Kind: ReqPing, - }, - wantConstruct: "/v2/", -}, { - testName: "ping", - method: "GET", - url: "/v2/", - wantRequest: &Request{ - Kind: ReqPing, - }, -}, { - testName: "getBlob", - method: "GET", - url: "/v2/foo/bar/blobs/sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", - wantRequest: &Request{ - Kind: ReqBlobGet, - Repo: "foo/bar", - Digest: "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", - }, -}, { - testName: "getBlobInvalidDigest", - method: "GET", - url: "/v2/foo/bar/blobs/sha256:wrong", - wantError: `digest invalid: badly formed digest`, -}, { - testName: "getBlobInvalidRepo", - method: "GET", - url: "/v2/foo/bAr/blobs/sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", - wantError: `name invalid: invalid repository name`, -}, { - testName: "startUpload", - method: "POST", - url: "/v2/somerepo/blobs/uploads/", - wantRequest: &Request{ - Kind: ReqBlobStartUpload, - Repo: "somerepo", - }, -}, { - testName: "uploadChunk", - method: "PATCH", - url: "/v2/somerepo/blobs/uploads/YmxhaGJsYWg", - wantRequest: &Request{ - Kind: ReqBlobUploadChunk, - Repo: "somerepo", - UploadID: "blahblah", - }, -}, { - testName: "uploadChunk", - method: "PATCH", - url: "/v2/somerepo/blobs/uploads/YmxhaGJsYWg", - wantRequest: &Request{ - Kind: ReqBlobUploadChunk, - Repo: "somerepo", - UploadID: "blahblah", - }, -}, { - testName: "badlyFormedUploadDigest", - method: "POST", - url: "/v2/foo/blobs/uploads?digest=sha256:fake", - wantError: `digest invalid: badly formed digest`, -}, { - testName: "getUploadInfo", - method: "GET", - url: "/v2/myorg/myrepo/blobs/uploads/YmxhaGJsYWg", - wantRequest: &Request{ - Kind: ReqBlobUploadInfo, - Repo: "myorg/myrepo", - UploadID: "blahblah", - }, -}, { - testName: "mount", - method: "POST", - url: "/v2/x/y/blobs/uploads/?mount=sha256:c659529df24a1878f6df8d93c652280235a50b95e862d8e5cb566ee5b9ed6386&from=somewhere/other", - wantRequest: &Request{ - Kind: ReqBlobMount, - Repo: "x/y", - Digest: "sha256:c659529df24a1878f6df8d93c652280235a50b95e862d8e5cb566ee5b9ed6386", - FromRepo: "somewhere/other", - }, -}, { - testName: "mount2", - method: "POST", - url: "/v2/myorg/other/blobs/uploads/?from=myorg%2Fmyrepo&mount=sha256%3Ad647b322fff1e9dcb828ee67a6c6d1ed0ceef760988fdf54f9cfdeb96186e001", - wantRequest: &Request{ - Kind: ReqBlobMount, - Repo: "myorg/other", - Digest: "sha256:d647b322fff1e9dcb828ee67a6c6d1ed0ceef760988fdf54f9cfdeb96186e001", - FromRepo: "myorg/myrepo", - }, -}, { - testName: "mountWithNoFrom", - method: "POST", - url: "/v2/x/y/blobs/uploads/?mount=sha256:c659529df24a1878f6df8d93c652280235a50b95e862d8e5cb566ee5b9ed6386", - wantRequest: &Request{ - Kind: ReqBlobStartUpload, - Repo: "x/y", - }, - wantConstruct: "/v2/x/y/blobs/uploads/", -}, { - testName: "manifestHead", - method: "HEAD", - url: "/v2/myorg/myrepo/manifests/sha256:681aef2367e055f33cb8a6ab9c3090931f6eefd0c3ef15c6e4a79bdadfdb8982", - wantRequest: &Request{ - Kind: ReqManifestHead, - Repo: "myorg/myrepo", - Digest: "sha256:681aef2367e055f33cb8a6ab9c3090931f6eefd0c3ef15c6e4a79bdadfdb8982", - }, -}} - -func TestParseRequest(t *testing.T) { - for _, test := range parseRequestTests { - t.Run(test.testName, func(t *testing.T) { - u, err := url.Parse(test.url) - if err != nil { - t.Fatal(err) - } - rreq, err := Parse(test.method, u) - if test.wantError != "" { - require.Error(t, err) - require.Regexp(t, test.wantError, err.Error()) - // TODO http code - return - } - require.NoError(t, err) - require.Equal(t, test.wantRequest, rreq) - method, ustr := rreq.MustConstruct() - if test.wantConstruct == "" { - test.wantConstruct = test.url - } - - assert.Equal(t, test.method, method) - assert.Equal(t, canonURL(test.wantConstruct), canonURL(ustr)) - }) - } -} - -func canonURL(ustr string) string { - u, err := url.Parse(ustr) - if err != nil { - panic(err) - } - qv := u.Query() - if len(qv) == 0 { - return ustr - } - u.RawQuery = qv.Encode() - return u.String() -} diff --git a/ociclient/auth_test.go b/ociclient/auth_test.go index a225684..a5e6492 100644 --- a/ociclient/auth_test.go +++ b/ociclient/auth_test.go @@ -25,7 +25,9 @@ func TestAuthScopes(t *testing.T) { // All the call semantics themselves are tested elsewhere, but we want to be // sure that we're passing the right required auth scopes to the authorizer. - srv := httptest.NewServer(ociserver.New(ocimem.New(), nil)) + handler, err := ociserver.New(ocimem.New(), nil) + require.NoError(t, err) + srv := httptest.NewServer(handler) defer srv.Close() srvURL, _ := url.Parse(srv.URL) diff --git a/ociclient/error_test.go b/ociclient/error_test.go index be7b581..919f61f 100644 --- a/ociclient/error_test.go +++ b/ociclient/error_test.go @@ -19,11 +19,13 @@ import ( func TestErrorStuttering(t *testing.T) { // This checks that the stuttering observed in issue #31 // isn't an issue when ociserver wraps ociclient. - srv := httptest.NewServer(ociserver.New(&oci.Funcs{ + handler, err := ociserver.New(&oci.Funcs{ NewError: func(ctx context.Context, methodName, repo string) error { return oci.ErrManifestUnknown }, - }, nil)) + }, nil) + require.NoError(t, err) + srv := httptest.NewServer(handler) defer srv.Close() srvURL, _ := url.Parse(srv.URL) @@ -33,7 +35,7 @@ func TestErrorStuttering(t *testing.T) { require.NoError(t, err) _, err = r.GetTag(context.Background(), "foo", "sometag") assert.ErrorIs(t, err, oci.ErrManifestUnknown) - assert.Regexp(t, `404 Not Found: manifest unknown: manifest unknown to registry`, err.Error()) + assert.Regexp(t, `404 Not Found: manifest unknown: manifest \(sometag\) unknown to registry`, err.Error()) // ResolveTag uses HEAD rather than GET, so here we're testing // the path where a response with no body gets turned back into diff --git a/ociclient/referrers_test.go b/ociclient/referrers_test.go index da7721e..15429ca 100644 --- a/ociclient/referrers_test.go +++ b/ociclient/referrers_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "net/http" "net/http/httptest" "net/url" "strings" @@ -25,8 +26,14 @@ func TestReferrersFallback(t *testing.T) { // Test that the client falls back to using the referrers tag API // when the referrers API is not enabled. - srv := httptest.NewServer(ociserver.New(ocidebug.New(ocimem.New(), t.Logf), &ociserver.Options{ - DisableReferrersAPI: true, + handler, err := ociserver.New(ocidebug.New(ocimem.New(), t.Logf), nil) + require.NoError(t, err) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/referrers/") { + http.NotFound(w, r) + return + } + handler.ServeHTTP(w, r) })) t.Cleanup(srv.Close) diff --git a/ociserver/blobs.go b/ociserver/blobs.go new file mode 100644 index 0000000..cc3d566 --- /dev/null +++ b/ociserver/blobs.go @@ -0,0 +1,174 @@ +package ociserver + +import ( + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + + "github.com/docker/oci" + "github.com/docker/oci/ocidigest" + "github.com/docker/oci/ociserver/mux" +) + +func (s *Server) blobHeadGet() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := mux.URLParam(r, "name") + dgstStr := mux.URLParam(r, "digest") + + dgst, err := ocidigest.Parse(dgstStr) + if err != nil { + returnError(w, ErrBlobUnknown("invalid digest")) + return + } + blob, err := s.db.ResolveBlob(r.Context(), name, dgst) + if err != nil { + if errors.Is(err, oci.ErrBlobUnknown) || errors.Is(err, oci.ErrNameUnknown) { + returnError(w, ErrBlobUnknown(dgst.String())) + return + } + s.logError(r.Context(), "resolving blob", err, "repository", name, "digest", dgstStr) + returnError(w, ErrServerError()) + return + } + + switch r.Method { + case http.MethodHead: + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", fmt.Sprintf("%d", blob.Size)) + w.Header().Set("Docker-Content-Digest", blob.Digest.String()) + // TODO: solve eventing. + case http.MethodGet: + if s.redirect != nil { + redirect, redirectURL, err := s.redirect.Redirect(r, blob, name) + if err != nil { + s.logError(r.Context(), "getting blob redirect", err, "repository", name, "digest", dgstStr) + returnError(w, ErrServerError()) + return + } + if redirect { + // TODO: solve eventing. + http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect) + return + } + } + rh := r.Header.Get("Range") + + var br io.ReadCloser + if rh != "" { + if !strings.HasPrefix(rh, "bytes=") { + returnError(w, ErrRangeNotSatisfiable("Range header must use bytes unit")) + return + } + rangeStart, rangeEnd, _ := strings.Cut(strings.TrimPrefix(rh, "bytes="), "-") + var bi, ei int64 + if rangeStart == "" { + // Suffix range (e.g. bytes=-500): return the last N bytes. + suffixLen, parseErr := strconv.ParseInt(rangeEnd, 10, 64) + if parseErr != nil || suffixLen < 0 { + returnError(w, ErrRangeNotSatisfiable("invalid range")) + return + } + bi = blob.Size - suffixLen + if bi < 0 { + bi = 0 + } + ei = blob.Size - 1 + } else { + var parseErr error + bi, parseErr = strconv.ParseInt(rangeStart, 10, 64) + if parseErr != nil || bi < 0 { + returnError(w, ErrRangeNotSatisfiable("invalid range start")) + return + } + if rangeEnd == "" { + ei = blob.Size - 1 + } else { + ei, parseErr = strconv.ParseInt(rangeEnd, 10, 64) + if parseErr != nil || ei < 0 { + returnError(w, ErrRangeNotSatisfiable("invalid range end")) + return + } + } + } + if ei >= blob.Size { + ei = blob.Size - 1 + } + if bi >= blob.Size { + returnError(w, ErrRangeNotSatisfiable("range start is beyond end of blob")) + return + } + if ei < bi { + returnError(w, ErrRangeNotSatisfiable("range end is before range start")) + return + } + br, err = s.db.GetBlobRange(r.Context(), name, dgst, bi, ei+1) + if err != nil { + s.logError(r.Context(), "getting blob range", err, "repository", name, "digest", dgstStr) + returnError(w, ErrServerError()) + return + } + length := ei - bi + 1 + w.Header().Set("Content-Length", fmt.Sprintf("%d", length)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", bi, ei, blob.Size)) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Docker-Content-Digest", blob.Digest.String()) + w.WriteHeader(http.StatusPartialContent) + } else { + br, err = s.db.GetBlob(r.Context(), name, dgst) + if err != nil { + s.logError(r.Context(), "getting blob", err, "repository", name, "digest", dgstStr) + returnError(w, ErrServerError()) + return + } + w.Header().Set("Content-Length", fmt.Sprintf("%d", blob.Size)) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Docker-Content-Digest", blob.Digest.String()) + } + + defer func() { + err = br.Close() + if err != nil { + s.logError(r.Context(), "closing blob reader", err, "repository", name, "digest", dgstStr) + } + }() + _, err = io.Copy(w, br) + if err != nil { + s.logError(r.Context(), "writing blob response", err, "repository", name, "digest", dgstStr) + return + } + // TODO: solve eventing. + } + } +} + +func (s *Server) blobDelete() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := mux.URLParam(r, "name") + dgstStr := mux.URLParam(r, "digest") + + dgst, err := ocidigest.Parse(dgstStr) + if err != nil { + returnError(w, ErrDigestInvalid("invalid digest")) + return + } + + err = s.db.DeleteBlob(r.Context(), name, dgst) + if err != nil { + if errors.Is(err, oci.ErrBlobUnknown) || errors.Is(err, oci.ErrNameUnknown) { + returnError(w, ErrBlobUnknown(dgst.String())) + return + } + if errors.Is(err, oci.ErrReferenced) || errors.Is(err, oci.ErrDenied) { + returnError(w, ErrMethodNotAllowed("blob is referenced by a manifest")) + return + } + s.logError(r.Context(), "deleting blob", err, "repository", name, "digest", dgstStr) + returnError(w, ErrServerError()) + return + } + w.WriteHeader(http.StatusAccepted) + } +} diff --git a/ociserver/blobuploads.go b/ociserver/blobuploads.go new file mode 100644 index 0000000..f5d74aa --- /dev/null +++ b/ociserver/blobuploads.go @@ -0,0 +1,375 @@ +package ociserver + +import ( + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + + "github.com/docker/oci" + "github.com/docker/oci/ocidigest" + "github.com/docker/oci/ociref" + "github.com/docker/oci/ociserver/mux" +) + +func (s *Server) blobUploadGet() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := mux.URLParam(r, "name") + session := mux.URLParam(r, "session") + if session == "" { + returnError(w, ErrBlobUploadInvalid("missing session")) + return + } + bw, err := s.db.PushBlobChunkedResume(r.Context(), name, session, -1, 0) + if err != nil { + if errors.Is(err, oci.ErrBlobUploadUnknown) { + returnError(w, ErrBlobUploadInvalid("session not found")) + return + } + s.logError(r.Context(), "getting upload session", err, "repository", name, "session", session) + returnError(w, ErrServerError()) + return + } + u := fmt.Sprintf("/v2/%s/blobs/uploads/%s", name, session) + w.Header().Set("Location", u) + w.Header().Set("Range", fmt.Sprintf("%d-%d", 0, bw.Size()-1)) // adjust size to 0-based + w.WriteHeader(http.StatusNoContent) + } +} + +func (s *Server) blobUploadDelete() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := mux.URLParam(r, "name") + session := mux.URLParam(r, "session") + if session == "" { + returnError(w, ErrBlobUploadInvalid("missing session")) + return + } + bw, err := s.db.PushBlobChunkedResume(r.Context(), name, session, -1, 0) + if err != nil { + if errors.Is(err, oci.ErrBlobUploadUnknown) { + returnError(w, ErrBlobUploadInvalid("session not found")) + return + } + s.logError(r.Context(), "getting upload session", err, "repository", name, "session", session) + returnError(w, ErrServerError()) + return + } + err = bw.Cancel() + if err != nil { + s.logError(r.Context(), "canceling upload session", err, "repository", name, "session", session) + returnError(w, ErrServerError()) + return + } + w.WriteHeader(http.StatusNoContent) + } +} + +func (s *Server) blobUploadPost() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := mux.URLParam(r, "name") + dgstString := r.URL.Query().Get("digest") + mount := r.URL.Query().Get("mount") + from := r.URL.Query().Get("from") + + if mount != "" && from != "" { + if !ociref.IsValidRepository(from) { + returnError(w, ErrBlobUploadInvalid("invalid from parameter")) + return + } + dgst, err := ocidigest.Parse(mount) + if err != nil { + returnError(w, ErrBlobUploadInvalid("invalid digest")) + return + } + blob, err := s.db.MountBlob(r.Context(), from, name, dgst) + if err != nil { + goto FALLBACK + } + // TODO: solve eventing. + u := fmt.Sprintf("/v2/%s/blobs/%s", name, blob.Digest.String()) + w.Header().Set("Location", u) + w.Header().Set("Docker-Content-Digest", blob.Digest.String()) + w.WriteHeader(http.StatusCreated) + return + } else if dgstString != "" { + dgst, err := ocidigest.Parse(dgstString) + if err != nil { + returnError(w, ErrBlobUploadInvalid("invalid digest")) + return + } + contentLength := r.Header.Get("Content-Length") + if contentLength == "" { + contentLength = "0" + } + length, err := strconv.Atoi(contentLength) + if err != nil || length < 0 { + returnError(w, ErrBlobUploadInvalid("unable to parse Content-Length")) + return + } + bw, err := s.db.PushBlobChunked(r.Context(), name, length) + if err != nil { + s.logError(r.Context(), "starting blob upload", err, "repository", name) + returnError(w, ErrServerError()) + return + } + _, err = io.Copy(bw, r.Body) + if err != nil { + s.logError(r.Context(), "writing blob upload", err, "repository", name) + returnError(w, ErrServerError()) + return + } + err = r.Body.Close() + if err != nil { + s.logError(r.Context(), "closing blob upload request body", err, "repository", name) + } + if err = bw.Close(); err != nil { + s.logError(r.Context(), "closing blob writer", err, "repository", name) + returnError(w, ErrServerError()) + return + } + desc, err := bw.Commit(dgst) + if err != nil { + if errors.Is(err, oci.ErrDigestInvalid) { + returnError(w, ErrDigestInvalid("digest does not match contents")) + return + } + s.logError(r.Context(), "committing blob upload", err, "repository", name, "digest", dgst) + returnError(w, ErrServerError()) + return + } + // TODO: solve eventing. + u := fmt.Sprintf("/v2/%s/blobs/%s", name, desc.Digest.String()) + w.Header().Set("Location", u) + w.Header().Set("Docker-Content-Digest", desc.Digest.String()) + w.WriteHeader(http.StatusCreated) + return + } + FALLBACK: + bw, err := s.db.PushBlobChunked(r.Context(), name, 0) + if err != nil { + s.logError(r.Context(), "creating blob upload session", err, "repository", name) + returnError(w, ErrServerError()) + return + } + u := fmt.Sprintf("/v2/%s/blobs/uploads/%s", name, bw.ID()) + w.Header().Set("Location", u) + w.Header().Set("Range", "0-0") + w.WriteHeader(http.StatusAccepted) + } +} + +func (s *Server) blobUploadPatch() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + session := mux.URLParam(r, "session") + if session == "" { + returnError(w, ErrBlobUploadInvalid("missing session")) + return + } + contentLength := r.Header.Get("Content-Length") + streaming := contentLength == "" + if streaming { + contentLength = "0" + } + length, err := strconv.Atoi(contentLength) + if err != nil || length < 0 { + returnError(w, ErrBlobUploadInvalid("unable to parse Content-Length")) + return + } + contentRange := r.Header.Get("Content-Range") + var rangeStart, rangeEnd, size int + if contentRange == "" { + rangeStart = 0 + rangeEnd = length + size = length + } else { + rangeStart, rangeEnd, size, err = parseRange(contentRange) + if err != nil { + returnError(w, ErrBlobUploadInvalid("unable to parse range")) + return + } + } + + if length > 0 && length != size { + returnError(w, ErrBlobUploadInvalid("Content-Length does not match Content-Range")) + return + } + name := mux.URLParam(r, "name") + + bw, err := s.db.PushBlobChunkedResume(r.Context(), name, session, int64(rangeStart), size) + if err != nil { + if errors.Is(err, oci.ErrRangeInvalid) { + returnError(w, ErrBlobUploadOutOfOrder()) + return + } + s.logError(r.Context(), "resuming blob upload", err, "repository", name, "session", session) + returnError(w, ErrServerError()) + return + } + if size > 0 || streaming { + n, err := io.Copy(bw, r.Body) + if err != nil { + if errors.Is(err, oci.ErrRangeInvalid) { + returnError(w, ErrBlobUploadOutOfOrder()) + return + } + s.logError(r.Context(), "writing blob upload chunk", err, "repository", name, "session", session) + if cancelErr := bw.Cancel(); cancelErr != nil { + s.logError(r.Context(), "canceling blob upload after write failure", cancelErr, "repository", name, "session", session) + } + returnError(w, ErrServerError()) + return + } + if streaming { + rangeEnd = rangeStart + int(n) + } + } + err = r.Body.Close() + if err != nil { + s.logError(r.Context(), "closing blob upload request body", err, "repository", name, "session", session) + } + if err = bw.Close(); err != nil { + s.logError(r.Context(), "closing blob writer", err, "repository", name, "session", session) + returnError(w, ErrServerError()) + return + } + u := fmt.Sprintf("/v2/%s/blobs/uploads/%s", name, session) + w.Header().Set("Location", u) + w.Header().Set("Range", fmt.Sprintf("0-%d", rangeEnd-1)) + w.WriteHeader(http.StatusAccepted) + } +} + +func (s *Server) blobUploadPut() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + session := mux.URLParam(r, "session") + if session == "" { + returnError(w, ErrBlobUploadInvalid("missing session")) + return + } + dgstString := r.URL.Query().Get("digest") + if dgstString == "" { + returnError(w, ErrDigestInvalid("missing digest")) + return + } + dgst, err := ocidigest.Parse(dgstString) + if err != nil { + returnError(w, ErrBlobUploadInvalid("invalid digest")) + return + } + name := mux.URLParam(r, "name") + contentLength := r.Header.Get("Content-Length") + + streaming := contentLength == "" + if streaming { + contentLength = "0" + } + if streaming || contentLength != "0" { + length, err := strconv.Atoi(contentLength) + if err != nil || length < 0 { + returnError(w, ErrBlobUploadInvalid("unable to parse Content-Length")) + return + } + contentRange := r.Header.Get("Content-Range") + var rangeStart, size int + if contentRange == "" { + rangeStart = 0 + size = length + } else { + rangeStart, _, size, err = parseRange(contentRange) + if err != nil { + returnError(w, ErrBlobUploadInvalid("unable to parse range")) + return + } + } + if length > 0 && length != size { + returnError(w, ErrBlobUploadInvalid("Content-Length does not match Content-Range")) + return + } + + bw, err := s.db.PushBlobChunkedResume(r.Context(), name, session, int64(rangeStart), size) + if err != nil { + if errors.Is(err, oci.ErrRangeInvalid) { + returnError(w, ErrBlobUploadOutOfOrder()) + return + } + s.logError(r.Context(), "resuming blob upload", err, "repository", name, "session", session) + returnError(w, ErrServerError()) + return + } + _, err = io.Copy(bw, r.Body) + if err != nil { + if errors.Is(err, oci.ErrRangeInvalid) { + returnError(w, ErrBlobUploadOutOfOrder()) + return + } + s.logError(r.Context(), "writing blob upload chunk", err, "repository", name, "session", session) + returnError(w, ErrServerError()) + return + } + err = r.Body.Close() + if err != nil { + s.logError(r.Context(), "closing blob upload request body", err, "repository", name, "session", session) + } + if err = bw.Close(); err != nil { + s.logError(r.Context(), "closing blob writer", err, "repository", name, "session", session) + returnError(w, ErrServerError()) + return + } + } + bw, err := s.db.PushBlobChunkedResume(r.Context(), name, session, -1, 0) + if err != nil { + if !errors.Is(err, oci.ErrBlobUploadUnknown) { + s.logError(r.Context(), "getting blob upload session for commit", err, "repository", name, "session", session) + } + returnError(w, ErrBlobUploadUnknown(session)) + return + } + _, err = bw.Commit(dgst) + if err != nil { + if errors.Is(err, oci.ErrDigestInvalid) { + returnError(w, ErrDigestInvalid("digest does not match contents")) + return + } + s.logError(r.Context(), "committing blob upload", err, "repository", name, "session", session, "digest", dgst) + returnError(w, ErrServerError()) + return + } + // TODO: solve eventing. + u := fmt.Sprintf("/v2/%s/blobs/%s", name, dgst.String()) + w.Header().Set("Location", u) + w.Header().Set("Docker-Content-Digest", dgst.String()) + w.WriteHeader(http.StatusCreated) + } +} + +// parseRange parses a Content-Range value of the form [bytes ]start-end[/total]. +// It returns (start, end+1, size, nil) where end+1 is the exclusive upper bound, +// so callers can use rangeEnd-1 consistently for the Range response header. +func parseRange(contentRange string) (int, int, int, error) { + contentRange = strings.TrimPrefix(contentRange, "bytes ") + if i := strings.IndexByte(contentRange, '/'); i >= 0 { + contentRange = contentRange[:i] + } + rangeStart, rangeEnd, ok := strings.Cut(contentRange, "-") + if !ok { + return 0, 0, 0, errors.New("unable to parse range header") + } + start, err := strconv.Atoi(rangeStart) + if err != nil { + return 0, 0, 0, errors.New("unable to parse range start") + } + end, err := strconv.Atoi(rangeEnd) + if err != nil { + return 0, 0, 0, errors.New("unable to parse range end") + } + if start < 0 || end < 0 { + return 0, 0, 0, errors.New("range values must be non-negative") + } + if end < start { + return 0, 0, 0, errors.New("range end must be greater than or equal to range start") + } + return start, end + 1, end - start + 1, nil +} diff --git a/ociserver/blobuploads_test.go b/ociserver/blobuploads_test.go new file mode 100644 index 0000000..b7b4100 --- /dev/null +++ b/ociserver/blobuploads_test.go @@ -0,0 +1,331 @@ +package ociserver + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/docker/oci" + "github.com/docker/oci/ocidigest" +) + +func TestParseRange(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + contentRange string + wantStart int + wantEndExclusive int + wantSize int + wantErr bool + }{ + { + name: "bare range", + contentRange: "0-4", + wantStart: 0, + wantEndExclusive: 5, + wantSize: 5, + }, + { + name: "bytes range with total", + contentRange: "bytes 5-9/20", + wantStart: 5, + wantEndExclusive: 10, + wantSize: 5, + }, + { + name: "negative start", + contentRange: "-1-4", + wantErr: true, + }, + { + name: "negative end", + contentRange: "1--4", + wantErr: true, + }, + { + name: "end before start", + contentRange: "5-3", + wantErr: true, + }, + { + name: "missing dash", + contentRange: "5", + wantErr: true, + }, + { + name: "non numeric start", + contentRange: "a-5", + wantErr: true, + }, + { + name: "non numeric end", + contentRange: "5-b", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + gotStart, gotEndExclusive, gotSize, err := parseRange(tt.contentRange) + if tt.wantErr { + if err == nil { + t.Fatalf("parseRange(%q) succeeded, want error", tt.contentRange) + } + return + } + if err != nil { + t.Fatalf("parseRange(%q) returned error: %v", tt.contentRange, err) + } + if gotStart != tt.wantStart || gotEndExclusive != tt.wantEndExclusive || gotSize != tt.wantSize { + t.Fatalf("parseRange(%q) = (%d, %d, %d), want (%d, %d, %d)", + tt.contentRange, + gotStart, + gotEndExclusive, + gotSize, + tt.wantStart, + tt.wantEndExclusive, + tt.wantSize, + ) + } + }) + } +} + +func TestBlobHeadGetRangeOpenEnded(t *testing.T) { + t.Parallel() + + dgst := ocidigest.FromBytes([]byte("0123456789")) + blob := "0123456789" + + tests := []struct { + name string + rangeHeader string + wantOffset0 int64 + wantOffset1 int64 + wantContentRange string + wantBody string + }{ + { + name: "missing start returns suffix range", + rangeHeader: "bytes=-4", + wantOffset0: 6, + wantOffset1: 10, + wantContentRange: "bytes 6-9/10", + wantBody: "6789", + }, + { + name: "missing end returns to end of blob", + rangeHeader: "bytes=4-", + wantOffset0: 4, + wantOffset1: 10, + wantContentRange: "bytes 4-9/10", + wantBody: "456789", + }, + { + name: "explicit range passes exclusive upper bound to storage", + rangeHeader: "bytes=2-5", + wantOffset0: 2, + wantOffset1: 6, + wantContentRange: "bytes 2-5/10", + wantBody: "2345", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + s := &Server{ + db: &oci.Funcs{ + ResolveBlob_: func(_ context.Context, repo string, gotDigest oci.Digest) (oci.Descriptor, error) { + if repo != "repo" { + t.Fatalf("ResolveBlob repo = %q, want repo", repo) + } + if gotDigest != dgst { + t.Fatalf("ResolveBlob digest = %q, want %q", gotDigest, dgst) + } + return oci.Descriptor{ + Digest: dgst, + Size: int64(len(blob)), + }, nil + }, + GetBlobRange_: func(_ context.Context, repo string, gotDigest oci.Digest, offset0, offset1 int64) (oci.BlobReader, error) { + if repo != "repo" { + t.Fatalf("GetBlobRange repo = %q, want repo", repo) + } + if gotDigest != dgst { + t.Fatalf("GetBlobRange digest = %q, want %q", gotDigest, dgst) + } + if offset0 != tt.wantOffset0 || offset1 != tt.wantOffset1 { + t.Fatalf("GetBlobRange offsets = (%d, %d), want (%d, %d)", offset0, offset1, tt.wantOffset0, tt.wantOffset1) + } + return &testBlobReader{ + ReadCloser: io.NopCloser(strings.NewReader(blob[offset0:offset1])), + desc: oci.Descriptor{ + Digest: dgst, + Size: int64(len(blob)), + }, + }, nil + }, + }, + } + req := httptest.NewRequest(http.MethodGet, "/v2/repo/blobs/"+dgst.String(), nil) + req.Header.Set("Range", tt.rangeHeader) + rec := httptest.NewRecorder() + + serveTestRoute(t, `/v2/*name/blobs/:digest`, s.blobHeadGet(), rec, req) + + if rec.Code != http.StatusPartialContent { + t.Fatalf("status = %d, want %d; body: %s", rec.Code, http.StatusPartialContent, rec.Body.String()) + } + if got := rec.Header().Get("Content-Range"); got != tt.wantContentRange { + t.Fatalf("Content-Range = %q, want %q", got, tt.wantContentRange) + } + if got := rec.Body.String(); got != tt.wantBody { + t.Fatalf("body = %q, want %q", got, tt.wantBody) + } + }) + } +} + +func TestBlobUploadPutStreamsBodyWithoutContentLength(t *testing.T) { + t.Parallel() + + body := []byte("sha512 final put body") + dgst := ocidigest.SHA512.FromBytes(body) + bw := &testBlobWriter{id: "session"} + var resumeOffsets []int64 + s := &Server{ + db: &oci.Funcs{ + // docker/oci currently checks PushBlobChunked_ before dispatching + // PushBlobChunkedResume_, so keep this non-nil in the test double. + PushBlobChunked_: func(context.Context, string, int) (oci.BlobWriter, error) { + return nil, nil + }, + PushBlobChunkedResume_: func(_ context.Context, repo, id string, offset int64, _ int) (oci.BlobWriter, error) { + if repo != "repo" { + t.Fatalf("repo = %q, want repo", repo) + } + if id != "session" { + t.Fatalf("session = %q, want session", id) + } + resumeOffsets = append(resumeOffsets, offset) + return bw, nil + }, + }, + } + req := httptest.NewRequest(http.MethodPut, "/v2/repo/blobs/uploads/session?digest="+dgst.String(), bytes.NewReader(body)) + req.Header.Del("Content-Length") + req.ContentLength = -1 + rec := httptest.NewRecorder() + + serveTestRoute(t, `/v2/*name/blobs/uploads/:session`, s.blobUploadPut(), rec, req) + + if rec.Code != http.StatusCreated { + t.Fatalf("status = %d, want %d; body: %s", rec.Code, http.StatusCreated, rec.Body.String()) + } + if got := bw.String(); got != string(body) { + t.Fatalf("written body = %q, want %q", got, body) + } + if bw.committedDigest != dgst { + t.Fatalf("committed digest = %s, want %s", bw.committedDigest, dgst) + } + if len(resumeOffsets) != 2 || resumeOffsets[0] != 0 || resumeOffsets[1] != -1 { + t.Fatalf("resume offsets = %v, want [0 -1]", resumeOffsets) + } +} + +func TestBlobUploadPatchMapsWriterRangeInvalidToOutOfOrder(t *testing.T) { + t.Parallel() + + s := &Server{ + db: &oci.Funcs{ + PushBlobChunked_: func(context.Context, string, int) (oci.BlobWriter, error) { + return nil, nil + }, + PushBlobChunkedResume_: func(context.Context, string, string, int64, int) (oci.BlobWriter, error) { + return &errorBlobWriter{err: oci.ErrRangeInvalid}, nil + }, + }, + } + req := httptest.NewRequest(http.MethodPatch, "/v2/repo/blobs/uploads/session", strings.NewReader("abc")) + req.Header.Set("Content-Length", "3") + req.Header.Set("Content-Range", "3-5") + rec := httptest.NewRecorder() + + serveTestRoute(t, `/v2/*name/blobs/uploads/:session`, s.blobUploadPatch(), rec, req) + + if rec.Code != http.StatusRequestedRangeNotSatisfiable { + t.Fatalf("status = %d, want %d; body: %s", rec.Code, http.StatusRequestedRangeNotSatisfiable, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "BLOB_UPLOAD_INVALID") { + t.Fatalf("body = %q, want BLOB_UPLOAD_INVALID", rec.Body.String()) + } +} + +type testBlobReader struct { + io.ReadCloser + desc oci.Descriptor +} + +func (r *testBlobReader) Descriptor() oci.Descriptor { + return r.desc +} + +type testBlobWriter struct { + bytes.Buffer + id string + committedDigest oci.Digest +} + +func (w *testBlobWriter) Close() error { return nil } + +func (w *testBlobWriter) Cancel() error { return nil } + +func (w *testBlobWriter) Size() int64 { return int64(w.Len()) } + +func (w *testBlobWriter) ChunkSize() int { return 0 } + +func (w *testBlobWriter) ID() string { return w.id } + +func (w *testBlobWriter) Commit(dgst oci.Digest) (oci.Descriptor, error) { + if dgst.Algorithm().FromBytes(w.Bytes()) != dgst { + return oci.Descriptor{}, oci.ErrDigestInvalid + } + w.committedDigest = dgst + return oci.Descriptor{ + Digest: dgst, + Size: int64(w.Len()), + }, nil +} + +type errorBlobWriter struct { + err error +} + +func (w *errorBlobWriter) Write([]byte) (int, error) { + return 0, w.err +} + +func (w *errorBlobWriter) Close() error { return nil } + +func (w *errorBlobWriter) Cancel() error { return nil } + +func (w *errorBlobWriter) Size() int64 { return 0 } + +func (w *errorBlobWriter) ChunkSize() int { return 0 } + +func (w *errorBlobWriter) ID() string { return "session" } + +func (w *errorBlobWriter) Commit(oci.Digest) (oci.Descriptor, error) { + return oci.Descriptor{}, errors.New("unexpected commit") +} diff --git a/ociserver/compatibility_test.go b/ociserver/compatibility_test.go deleted file mode 100644 index 9d155be..0000000 --- a/ociserver/compatibility_test.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ociserver_test - -// -//import ( -// "bytes" -// "net/http/httptest" -// "strings" -// "testing" -// -// "github.com/google/go-containerregistry/pkg/name" -// "github.com/google/go-containerregistry/pkg/registry" -// "github.com/google/go-containerregistry/pkg/v1/random" -// "github.com/google/go-containerregistry/pkg/v1/remote" -// "github.com/google/go-containerregistry/pkg/v1/tarball" -//) -// -//func TestPushAndPullContainer(t *testing.T) { -// s := httptest.NewServer(registry.New()) -// defer s.Close() -// -// r := strings.TrimPrefix(s.URL, "http://") + "/foo:latest" -// d, err := name.NewTag(r) -// if err != nil { -// t.Fatalf("Unable to create tag: %v", err) -// } -// -// i, err := random.Image(1024, 1) -// if err != nil { -// t.Fatalf("Unable to make random image: %v", err) -// } -// -// if err := remote.Write(d, i); err != nil { -// t.Fatalf("Error writing image: %v", err) -// } -// -// ref, err := name.ParseReference(r) -// if err != nil { -// t.Fatalf("Error parsing tag: %v", err) -// } -// -// ri, err := remote.Image(ref) -// if err != nil { -// t.Fatalf("Error reading image: %v", err) -// } -// -// b := &bytes.Buffer{} -// if err := tarball.Write(ref, ri, b); err != nil { -// t.Fatalf("Error writing image to tarball: %v", err) -// } -//} diff --git a/ociserver/deleter.go b/ociserver/deleter.go deleted file mode 100644 index db12ad3..0000000 --- a/ociserver/deleter.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ociserver - -import ( - "context" - "net/http" - - "github.com/docker/oci/internal/ocirequest" - "github.com/docker/oci/ocidigest" -) - -func (r *registry) handleBlobDelete(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - digest, err := ocidigest.Parse(rreq.Digest) - if err != nil { - return err - } - if err := r.backend.DeleteBlob(ctx, rreq.Repo, digest); err != nil { - return err - } - resp.WriteHeader(http.StatusAccepted) - return nil -} - -func (r *registry) handleManifestDelete(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - var err error - if rreq.Tag != "" { - err = r.backend.DeleteTag(ctx, rreq.Repo, rreq.Tag) - } else { - digest, parseErr := ocidigest.Parse(rreq.Digest) - if parseErr != nil { - return parseErr - } - err = r.backend.DeleteManifest(ctx, rreq.Repo, digest) - } - if err != nil { - return err - } - resp.WriteHeader(http.StatusAccepted) - return nil -} diff --git a/ociserver/error.go b/ociserver/error.go deleted file mode 100644 index d02b2b3..0000000 --- a/ociserver/error.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ociserver - -import ( - "fmt" - - "github.com/docker/oci" -) - -func withHTTPCode(statusCode int, err error) error { - return oci.NewHTTPError(err, statusCode, nil, nil) -} - -func badAPIUseError(f string, a ...any) error { - return oci.NewError(fmt.Sprintf(f, a...), oci.ErrUnsupported.Code(), nil) -} diff --git a/ociserver/error_test.go b/ociserver/error_test.go deleted file mode 100644 index 3f6c8ef..0000000 --- a/ociserver/error_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2023 CUE Labs AG -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ociserver - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - - "github.com/docker/oci" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCustomErrorWriter(t *testing.T) { - // Test that if an Interface method returns an HTTPError error, the - // HTTP status code is derived from the OCI error code in preference - // to the HTTPError status code. - r := New(&oci.Funcs{}, &Options{ - WriteError: func(w http.ResponseWriter, _ *http.Request, err error) { - w.Header().Set("Some-Header", "a value") - oci.WriteError(w, err) - }, - }) - s := httptest.NewServer(r) - defer s.Close() - resp, err := http.Get(s.URL + "/v2/foo/manifests/sometag") - require.NoError(t, err) - defer resp.Body.Close() - require.Equal(t, "a value", resp.Header.Get("Some-Header")) -} - -func TestHTTPStatusOverriddenByErrorCode(t *testing.T) { - // Test that if an Interface method returns an HTTPError error, the - // HTTP status code is derived from the OCI error code in preference - // to the HTTPError status code. - r := New(&oci.Funcs{ - GetTag_: func(ctx context.Context, repo string, tagName string) (oci.BlobReader, error) { - return nil, oci.NewHTTPError(oci.ErrNameUnknown, http.StatusUnauthorized, nil, nil) - }, - }, nil) - s := httptest.NewServer(r) - defer s.Close() - resp, err := http.Get(s.URL + "/v2/foo/manifests/sometag") - require.NoError(t, err) - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - require.Equal(t, http.StatusNotFound, resp.StatusCode) - expected := &oci.WireErrors{ - Errors: []oci.WireError{{ - Code_: oci.ErrNameUnknown.Code(), - Message: "401 Unauthorized: name unknown: repository name not known to registry", - }}, - } - expectedJSON, err := json.Marshal(expected) - require.NoError(t, err) - assert.JSONEq(t, string(expectedJSON), string(body)) -} - -func TestHTTPStatusUsedForUnknownErrorCode(t *testing.T) { - // Test that if an Interface method returns an HTTPError error, that - // HTTP status code is used when the code isn't known to be - // associated with a particular HTTP status. - r := New(&oci.Funcs{ - GetTag_: func(ctx context.Context, repo string, tagName string) (oci.BlobReader, error) { - return nil, oci.NewHTTPError(oci.NewError("foo", "SOMECODE", nil), http.StatusTeapot, nil, nil) - }, - }, nil) - s := httptest.NewServer(r) - defer s.Close() - resp, err := http.Get(s.URL + "/v2/foo/manifests/sometag") - require.NoError(t, err) - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - require.Equal(t, http.StatusTeapot, resp.StatusCode) - expected := &oci.WireErrors{ - Errors: []oci.WireError{{ - Code_: "SOMECODE", - Message: "foo", - }}, - } - expectedJSON, err := json.Marshal(expected) - require.NoError(t, err) - assert.JSONEq(t, string(expectedJSON), string(body)) -} diff --git a/ociserver/errors.go b/ociserver/errors.go new file mode 100644 index 0000000..310f529 --- /dev/null +++ b/ociserver/errors.go @@ -0,0 +1,228 @@ +package ociserver + +import ( + "fmt" + "net/http" +) + +// OCIError is a struct that implements the error interface, and formats errors in a way that adheres +// to the OCI distribution spec +type OCIError struct { + status int + Code string `json:"code"` + Message string `json:"message,omitempty"` + Detail string `json:"detail,omitempty"` +} + +// Error implements the error interface +func (e *OCIError) Error() string { + return fmt.Sprintf("%s-%s", e.Code, e.Message) +} + +// Status returns the http status code associated with the error +func (e *OCIError) Status() int { + return e.status +} + +// ErrBlobUnknown is for when a blob is not found in the registry +func ErrBlobUnknown(digest string) *OCIError { + return &OCIError{ + status: http.StatusNotFound, + Code: "BLOB_UNKNOWN", + Message: fmt.Sprintf("blob (%s) unknown to registry", digest), + } +} + +// ErrBlobUploadInvalid is a generic error message if something is invalid on upload +func ErrBlobUploadInvalid(msg string) *OCIError { + return &OCIError{ + status: http.StatusBadRequest, + Code: "BLOB_UPLOAD_INVALID", + Message: fmt.Sprintf("blob upload invalid: %s", msg), + } +} + +// ErrBlobUploadOutOfOrder is an error when blobs are uploaded out of order +func ErrBlobUploadOutOfOrder() *OCIError { + return &OCIError{ + status: http.StatusRequestedRangeNotSatisfiable, + Code: "BLOB_UPLOAD_INVALID", + Message: "upload out of order", + } +} + +// ErrBlobUploadUnknown is an error for when the upload session is not found +func ErrBlobUploadUnknown(session string) *OCIError { + return &OCIError{ + status: http.StatusNotFound, + Code: "BLOB_UPLOAD_UNKNOWN", + Message: fmt.Sprintf("blob upload (%s) unknown to registry", session), + } +} + +// ErrDigestInvalid is sent if the digest is invalid +func ErrDigestInvalid(msg string) *OCIError { + return &OCIError{ + status: http.StatusBadRequest, + Code: "DIGEST_INVALID", + Message: fmt.Sprintf("digest invalid: %s", msg), + } +} + +// ErrManifestBlobUnknown is an error for when the registry has not received a blob referenced by a manifest +func ErrManifestBlobUnknown(digest string) *OCIError { + return &OCIError{ + status: http.StatusNotFound, + Code: "MANIFEST_BLOB_UNKNOWN", + Message: fmt.Sprintf("referenced manifest or blob (%s) unknown to registry", digest), + } +} + +// ErrManifestInvalid is an error for when a manifest is invalid in some way +func ErrManifestInvalid(details string) *OCIError { + return &OCIError{ + status: http.StatusBadRequest, + Code: "MANIFEST_INVALID", + Message: fmt.Sprintf("manifest is invalid: %s", details), + } +} + +// ErrManifestUnknown is an error for if a manifest is not found in the registry +func ErrManifestUnknown(digest string) *OCIError { + return &OCIError{ + status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: fmt.Sprintf("manifest (%s) unknown to registry", digest), + } +} + +// ErrReferrersUnknown is an error for if there are no referrers for a given digest +func ErrReferrersUnknown(digest string) *OCIError { + return &OCIError{ + status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: fmt.Sprintf("referrers (%s) unknown to registry", digest), + } +} + +// ErrMediaTypeUnsupported is an error for when a pushed mediatype is unsupported +func ErrMediaTypeUnsupported(mediaType string) *OCIError { + return &OCIError{ + status: http.StatusUnsupportedMediaType, + Code: "UNSUPPORTED", + Message: fmt.Sprintf("unsupported mediatype (%s)", mediaType), + } +} + +// ErrNotAcceptable is returned when a stored representation cannot satisfy the Accept header. +func ErrNotAcceptable(mediaType string) *OCIError { + return &OCIError{ + status: http.StatusNotAcceptable, + Code: "UNSUPPORTED", + Message: fmt.Sprintf("requested manifest mediatype is not acceptable (%s)", mediaType), + } +} + +// ErrNameUnknown is returned when the repository name is not known to the registry +func ErrNameUnknown(name string) *OCIError { + return &OCIError{ + status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: fmt.Sprintf("repository name not known to registry: %s", name), + } +} + +// ErrNameInvalid is returned when the repository name is invalid. +func ErrNameInvalid(name string) *OCIError { + return &OCIError{ + status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: fmt.Sprintf("repository name is invalid: %s", name), + } +} + +// SIZE_INVALID + +// ErrUnauthorized is a generic error for if a request is unauthorized +func ErrUnauthorized() *OCIError { + return &OCIError{ + status: http.StatusUnauthorized, + Code: "UNAUTHORIZED", + Message: "authentication is required", + } +} + +// ErrDenied is an error for when access is denied +func ErrDenied(msg string) *OCIError { + return &OCIError{ + status: http.StatusForbidden, + Code: "DENIED", + Message: "request access is denied", + Detail: msg, + } +} + +// UNSUPPORTED + +// ErrTooManyRequests is an error if too many requests have been sent +func ErrTooManyRequests() *OCIError { + return &OCIError{ + status: http.StatusTooManyRequests, + Code: "TOOMANYREQUESTS", + Message: "too many requests", + } +} + +// ErrRangeNotSatisfiable is returned when a requested byte range cannot be satisfied. +func ErrRangeNotSatisfiable(msg string) *OCIError { + return &OCIError{ + status: http.StatusRequestedRangeNotSatisfiable, + Code: "UNSUPPORTED", + Message: msg, + } +} + +// ErrServerError is returned when an unexpected internal error occurs. +func ErrServerError() *OCIError { + return &OCIError{ + status: http.StatusInternalServerError, + Code: "SERVER_ERROR", + Message: "internal server error", + } +} + +// ErrBadRequest is not an OCI error... just added for a catch-all for now +func ErrBadRequest(msg string) *OCIError { + return &OCIError{ + status: http.StatusBadRequest, + Code: "BAD_REQUEST", + Message: msg, + } +} + +// ErrMethodNotAllowed is returned when an operation is not permitted on the target resource. +func ErrMethodNotAllowed(msg string) *OCIError { + return &OCIError{ + status: http.StatusMethodNotAllowed, + Code: "UNSUPPORTED", + Message: msg, + } +} + +// ErrManifestTooLarge is returned when the pushed manifest exceeds the registry size limit. +func ErrManifestTooLarge() *OCIError { + return &OCIError{ + status: http.StatusRequestEntityTooLarge, + Code: "MANIFEST_INVALID", + Message: "manifest exceeds maximum allowed size", + } +} + +// ErrNotImplemented is returned when an endpoint or operation has not been implemented. +func ErrNotImplemented() *OCIError { + return &OCIError{ + status: http.StatusNotImplemented, + Code: "UNSUPPORTED", + Message: "operation is not implemented", + } +} diff --git a/ociserver/lister.go b/ociserver/lister.go deleted file mode 100644 index cd9818d..0000000 --- a/ociserver/lister.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ociserver - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "iter" - "net/http" - "net/url" - "strconv" - - "github.com/docker/oci" - - "github.com/docker/oci/internal/ocirequest" - "github.com/docker/oci/ocidigest" -) - -const maxPageSize = 10000 - -type catalog struct { - Repos []string `json:"repositories"` -} - -type listTags struct { - Name string `json:"name"` - Tags []string `json:"tags"` -} - -func (r *registry) handleTagsList(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - tags, link, err := r.nextListResults(req, rreq, r.backend.Tags(ctx, rreq.Repo, &oci.TagsParameters{ - StartAfter: rreq.ListLast, - Limit: rreq.ListN, - })) - if err != nil { - return err - } - msg, _ := json.Marshal(listTags{ - Name: rreq.Repo, - Tags: tags, - }) - if link != "" { - resp.Header().Set("Link", link) - } - resp.Header().Set("Content-Length", strconv.Itoa(len(msg))) - resp.WriteHeader(http.StatusOK) - resp.Write(msg) - return nil -} - -func (r *registry) handleCatalogList(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) (_err error) { - repos, link, err := r.nextListResults(req, rreq, r.backend.Repositories(ctx, rreq.ListLast)) - if err != nil { - return err - } - msg, err := json.Marshal(catalog{ - Repos: repos, - }) - if err != nil { - return err - } - if link != "" { - resp.Header().Set("Link", link) - } - resp.Header().Set("Content-Length", strconv.Itoa(len(msg))) - resp.WriteHeader(http.StatusOK) - io.Copy(resp, bytes.NewReader(msg)) - return nil -} - -func (r *registry) handleReferrersList(ctx context.Context, resp http.ResponseWriter, _ *http.Request, rreq *ocirequest.Request) error { - if r.opts.DisableReferrersAPI { - return withHTTPCode(http.StatusNotFound, fmt.Errorf("referrers API has been disabled")) - } - artifactType := rreq.ArtifactType - if r.opts.DisableReferrersFiltering { - artifactType = "" - } - - im := &oci.IndexOrManifest{ - SchemaVersion: 2, - MediaType: mediaTypeOCIImageIndex, - } - - // TODO this could potentially end up with a very large response which we might - // want to limit. The spec does provide with a means to let a server respond with a partial - // request, linked to the next one with a Link header. However, arranging that is non-trivial - // because we'd need a way to return a link value to the client that enables a fresh - // call to Referrers to start where the old one left off. For now, we'll punt. - digest, err := ocidigest.Parse(rreq.Digest) - if err != nil { - return err - } - for desc, err := range r.backend.Referrers(ctx, rreq.Repo, digest, &oci.ReferrersParameters{ - ArtifactType: artifactType, - }) { - if err != nil { - return err - } - im.Manifests = append(im.Manifests, desc) - } - msg, err := json.Marshal(im) - if err != nil { - return err - } - resp.Header().Set("Content-Length", strconv.Itoa(len(msg))) - resp.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json") - if artifactType != "" { - resp.Header().Set("OCI-Filters-Applied", "artifactType") - } - resp.WriteHeader(http.StatusOK) - resp.Write(msg) - return nil -} - -func (r *registry) nextListResults(req *http.Request, rreq *ocirequest.Request, itemsIter iter.Seq2[string, error]) (items []string, link string, _ error) { - if r.opts.MaxListPageSize > 0 && rreq.ListN > r.opts.MaxListPageSize { - return nil, "", oci.NewError(fmt.Sprintf("query parameter n is too large (n=%d, max=%d)", rreq.ListN, r.opts.MaxListPageSize), oci.ErrUnsupported.Code(), nil) - } - n := rreq.ListN - if n <= 0 { - n = maxPageSize - } - truncated := false - for item, err := range itemsIter { - if err != nil { - return nil, "", err - } - if rreq.ListN > 0 && len(items) >= rreq.ListN { - truncated = true - break - } - // TODO we might want some way to limit on the total number - // of items returned in the absence of a ListN limit. - items = append(items, item) - // TODO sanity check that the items are in lexical order? - } - if truncated && !r.opts.OmitLinkHeaderFromResponses { - link = r.makeNextLink(req, items[len(items)-1]) - } - return items, link, nil -} - -// makeNextLink returns an RFC 5988 Link value suitable for -// providing the next URL in a chain of list page results, -// starting after the given "startAfter" item. -// TODO this assumes that req.URL.Path is the actual -// path that the client used to access the server. This might -// not necessarily be true, so maybe it would be better to -// use a path-relative URL instead, although that's trickier -// to arrange. -func (r *registry) makeNextLink(req *http.Request, startAfter string) string { - // Use the "next" relation type: - // See https://html.spec.whatwg.org/multipage/links.html#link-type-next - query := req.URL.Query() - query.Set("last", startAfter) - u := &url.URL{ - Path: req.URL.Path, - RawQuery: query.Encode(), - } - return fmt.Sprintf(`<%v>;rel="next"`, u) -} diff --git a/ociserver/manifests.go b/ociserver/manifests.go new file mode 100644 index 0000000..055d45a --- /dev/null +++ b/ociserver/manifests.go @@ -0,0 +1,287 @@ +package ociserver + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "strconv" + "strings" + + "github.com/docker/oci" + "github.com/docker/oci/ocidigest" + "github.com/docker/oci/ociref" + "github.com/docker/oci/ociserver/mux" +) + +const manifestSizeLimit = 4 * 1024 * 1024 // 4 MB + +func (s *Server) manifestHeadGet() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := mux.URLParam(r, "name") + reference := mux.URLParam(r, "reference") + var desc oci.Descriptor + var err error + if strings.Contains(reference, ":") { // is digest + var dgst oci.Digest + dgst, err = ocidigest.Parse(reference) + if err != nil { + returnError(w, ErrManifestInvalid("invalid digest")) + return + } + desc, err = s.db.ResolveManifest(r.Context(), name, dgst) + } else { + desc, err = s.db.ResolveTag(r.Context(), name, reference) + } + if err != nil { + if errors.Is(err, oci.ErrManifestUnknown) || errors.Is(err, oci.ErrNameUnknown) { + returnError(w, ErrManifestUnknown(reference)) + return + } + s.logError(r.Context(), "resolving manifest", err, "repository", name, "reference", reference) + returnError(w, ErrServerError()) + return + } + + if !acceptsMediaType(r.Header.Values("Accept"), desc.MediaType) { + returnError(w, ErrNotAcceptable(desc.MediaType)) + return + } + + if r.Method == http.MethodHead { + w.Header().Set("Content-Type", desc.MediaType) + w.Header().Set("Content-Length", fmt.Sprintf("%d", desc.Size)) + w.Header().Set("Docker-Content-Digest", desc.Digest.String()) + // TODO: solve eventing. + return + } + + b, err := s.db.GetManifest(r.Context(), name, desc.Digest) + if err != nil { + s.logError(r.Context(), "getting manifest", err, "repository", name, "reference", reference) + returnError(w, ErrServerError()) + return + } + defer func() { + if err := b.Close(); err != nil { + s.logError(r.Context(), "closing manifest reader", err, "repository", name, "reference", reference) + } + }() + w.Header().Set("Content-Type", desc.MediaType) + w.Header().Set("Content-Length", fmt.Sprintf("%d", desc.Size)) + w.Header().Set("Docker-Content-Digest", desc.Digest.String()) + if i, err := io.Copy(w, b); err != nil { + s.logError(r.Context(), "writing manifest response", err, "repository", name, "reference", reference, "bytesWritten", i, "size", desc.Size) + } + // TODO: solve eventing. + } +} + +func acceptsMediaType(acceptHeaders []string, mediaType string) bool { + if len(acceptHeaders) == 0 || mediaType == "" { + return true + } + for _, acceptHeader := range acceptHeaders { + if acceptHeader == "" { + continue + } + for _, part := range strings.Split(acceptHeader, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + acceptedType, params, err := mime.ParseMediaType(part) + if err != nil { + acceptedType, _, _ = strings.Cut(part, ";") + acceptedType = strings.TrimSpace(acceptedType) + params = nil + } + if acceptedType == "" { + continue + } + if q, ok := params["q"]; ok { + qv, err := strconv.ParseFloat(q, 64) + if err == nil && qv <= 0 { + continue + } + } + if mediaTypeMatches(acceptedType, mediaType) { + return true + } + } + } + return false +} + +func mediaTypeMatches(acceptedType, mediaType string) bool { + if acceptedType == "*/*" || acceptedType == mediaType { + return true + } + acceptedKind, acceptedSubType, ok := strings.Cut(acceptedType, "/") + if !ok { + return false + } + mediaKind, _, ok := strings.Cut(mediaType, "/") + if !ok { + return false + } + return acceptedSubType == "*" && acceptedKind == mediaKind +} + +func (s *Server) manifestPut() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := mux.URLParam(r, "name") + reference := mux.URLParam(r, "reference") + tags := r.URL.Query()["tag"] + + defer func() { + err := r.Body.Close() + if err != nil { + s.logError(r.Context(), "closing manifest request body", err, "repository", name, "reference", reference) + } + }() + lr := io.LimitReader(r.Body, manifestSizeLimit+1) + b, err := io.ReadAll(lr) + if err != nil { + s.logError(r.Context(), "reading manifest request body", err, "repository", name, "reference", reference) + returnError(w, ErrServerError()) + return + } + if len(b) > manifestSizeLimit { + returnError(w, ErrManifestTooLarge()) + return + } + var dgst oci.Digest + var tag string + if alg, _, ok := strings.Cut(reference, ":"); ok { // is digest + algo, err := ocidigest.LookupAlgorithm(alg) + if err != nil { + returnError(w, ErrDigestInvalid("unavailable algorithm")) + return + } + dgst = algo.FromBytes(b) + if dgst.String() != reference { + returnError(w, ErrDigestInvalid(fmt.Sprintf("provided digest (%s) does not match content (%s)", reference, dgst.String()))) + return + } + } else { + if !ociref.IsValidTag(reference) { + returnError(w, ErrManifestInvalid("invalid tag name")) + return + } + tag = reference + dgst = ocidigest.FromBytes(b) + } + var mani oci.IndexOrManifest + err = json.Unmarshal(b, &mani) + if err != nil { + returnError(w, ErrManifestInvalid("unable to parse manifest")) + return + } + contentType := r.Header.Get("Content-Type") + if contentType != "" { + contentType, _, _ = strings.Cut(contentType, ";") // strip any parameters + } + if mani.MediaType != "" && contentType != "" && mani.MediaType != contentType { + returnError(w, ErrManifestInvalid("mediaType does not match Content-Type")) + return + } else if contentType == "" && mani.MediaType != "" { + contentType = mani.MediaType + } else if contentType == "" { + contentType = oci.MediaTypeImageManifest // default to OCI manifest + } + switch contentType { + case oci.MediaTypeImageIndex, oci.MediaTypeDockerManifestList, + oci.MediaTypeImageManifest, oci.MediaTypeDockerManifest: + default: + returnError(w, ErrMediaTypeUnsupported(contentType)) + return + } + + if mani.MediaType != contentType { + returnError(w, ErrManifestInvalid("contentType does not match")) + } + + if err = mani.Validate(); err != nil { + returnError(w, ErrManifestInvalid(err.Error())) + return + } + + if tag != "" { + tags = append(tags, tag) + } + params := &oci.PushManifestParameters{ + Digest: dgst, + Tags: tags, + } + _, err = s.db.PushManifest(r.Context(), name, b, contentType, params) + if err != nil { + if errors.Is(err, oci.ErrManifestBlobUnknown) { + returnError(w, ErrManifestBlobUnknown(err.Error())) + return + } + if errors.Is(err, oci.ErrSizeInvalid) || errors.Is(err, oci.ErrManifestInvalid) { + returnError(w, ErrManifestInvalid(err.Error())) + return + } + if errors.Is(err, oci.ErrDigestInvalid) { + returnError(w, ErrDigestInvalid("digest does not match contents")) + return + } + s.logError(r.Context(), "pushing manifest", err, "repository", name, "reference", reference, "digest", dgst) + returnError(w, ErrServerError()) + return + } + // TODO: solve eventing. + if mani.Subject != nil { + // Per OCI spec, a subject manifest need not exist at push time; set the header unconditionally. + w.Header().Set("OCI-Subject", mani.Subject.Digest.String()) + } + u := fmt.Sprintf("/v2/%s/manifests/%s", name, reference) + w.Header().Set("Location", u) + w.Header().Set("Docker-Content-Digest", dgst.String()) + for _, t := range tags { + if strings.ContainsAny(t, "\r\n") { + continue + } + w.Header().Add("OCI-Tag", t) + } + w.WriteHeader(http.StatusCreated) + } +} + +func (s *Server) manifestDelete() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := mux.URLParam(r, "name") + reference := mux.URLParam(r, "reference") + var err error + if strings.Contains(reference, ":") { + var dgst oci.Digest + dgst, err = ocidigest.Parse(reference) + if err != nil { + returnError(w, ErrManifestInvalid("invalid digest")) + return + } + err = s.db.DeleteManifest(r.Context(), name, dgst) + } else { + err = s.db.DeleteTag(r.Context(), name, reference) + } + if err != nil { + if errors.Is(err, oci.ErrManifestUnknown) || errors.Is(err, oci.ErrNameUnknown) { + returnError(w, ErrManifestUnknown(reference)) + return + } + if errors.Is(err, oci.ErrReferenced) { + returnError(w, ErrMethodNotAllowed("manifest is referenced by another manifest")) + return + } + s.logError(r.Context(), "deleting manifest", err, "repository", name, "reference", reference) + returnError(w, ErrServerError()) + return + } + // TODO: solve eventing. + w.WriteHeader(http.StatusAccepted) + } +} diff --git a/ociserver/manifests_test.go b/ociserver/manifests_test.go new file mode 100644 index 0000000..4750240 --- /dev/null +++ b/ociserver/manifests_test.go @@ -0,0 +1,78 @@ +package ociserver + +import "testing" + +func TestAcceptsMediaType(t *testing.T) { + t.Parallel() + + const manifestMediaType = "application/vnd.oci.image.manifest.v1+json" + + tests := []struct { + name string + acceptHeader []string + want bool + }{ + { + name: "empty accept allows stored type", + want: true, + }, + { + name: "exact match", + acceptHeader: []string{manifestMediaType}, + want: true, + }, + { + name: "exact match with parameters", + acceptHeader: []string{manifestMediaType + "; q=0.8"}, + want: true, + }, + { + name: "multiple values with match", + acceptHeader: []string{"application/vnd.oci.image.index.v1+json, " + manifestMediaType}, + want: true, + }, + { + name: "multiple header lines with match", + acceptHeader: []string{ + "application/vnd.oci.image.index.v1+json", + manifestMediaType, + }, + want: true, + }, + { + name: "any type wildcard", + acceptHeader: []string{"*/*"}, + want: true, + }, + { + name: "subtype wildcard", + acceptHeader: []string{"application/*"}, + want: true, + }, + { + name: "quality zero rejects otherwise matching type", + acceptHeader: []string{manifestMediaType + "; q=0"}, + want: false, + }, + { + name: "different type", + acceptHeader: []string{"application/vnd.oci.image.index.v1+json"}, + want: false, + }, + { + name: "different top level wildcard", + acceptHeader: []string{"text/*"}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := acceptsMediaType(tt.acceptHeader, manifestMediaType); got != tt.want { + t.Fatalf("acceptsMediaType(%q, %q) = %v, want %v", tt.acceptHeader, manifestMediaType, got, tt.want) + } + }) + } +} diff --git a/ociserver/mediatype.go b/ociserver/mediatype.go deleted file mode 100644 index cd5541b..0000000 --- a/ociserver/mediatype.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2023 CUE Labs AG -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ociserver - -import "github.com/docker/oci" - -const ( - mediaTypeOCIImageIndex = oci.MediaTypeImageIndex - mediaTypeOCIManifestSchema1 = oci.MediaTypeImageManifest - mediaTypeOCIConfigJSON = oci.MediaTypeImageConfig - mediaTypeDockerConfigJSON = "application/vnd.docker.container.image.v1+json" - mediaTypeOctetStream = "application/octet-stream" -) diff --git a/ociserver/mux/mux.go b/ociserver/mux/mux.go new file mode 100644 index 0000000..a5abb86 --- /dev/null +++ b/ociserver/mux/mux.go @@ -0,0 +1,489 @@ +package mux + +import ( + "context" + "fmt" + "net/http" + "regexp" + "strings" +) + +var _ Router = &Mux{} + +// methodAll is the internal wildcard method key used by Handle and HandleFunc +// to register a handler for every HTTP method. It is "*" rather than a word +// like "all" so it cannot be confused with, or shadowed by, a real HTTP method +// name (which Method normalizes to upper case). +const methodAll = "*" + +// contextKey is an unexported type used for the router's own context keys so +// they cannot collide with keys defined in other packages. +type contextKey int + +const ( + // ctxKeyRequestPath carries the remaining path a sub-Router should match + // against, set by Route before delegating to the sub-Router. + ctxKeyRequestPath contextKey = iota + + // ctxKeyParams carries named regular-expression captures. + ctxKeyParams +) + +// URLParam returns the value of the named regex capture group for the current +// request, or "" if no such group matched. +func URLParam(r *http.Request, name string) string { + return URLParamFromCtx(r.Context(), name) +} + +// URLParamFromCtx returns the value of the named regex capture group stored in +// ctx, or "" if no such group matched. +func URLParamFromCtx(ctx context.Context, name string) string { + params, _ := ctx.Value(ctxKeyParams).(map[string]string) + return params[name] +} + +// Mux routes HTTP requests using path templates. +type Mux struct { + // Custom method not allowed handler + methodNotAllowedHandler http.HandlerFunc + + // A reference to the parent mux used by subrouters when mounting + // to a parent mux + parent *Mux + + // Custom route not found handler + notFoundHandler http.HandlerFunc + + // Debug logger; nil means fall back to the parent's, then a no-op. Set via + // WithLogger. Resolved through log(). + logger Logger + + // The middleware stack + middlewares []func(http.Handler) http.Handler + + // Controls the behaviour of middleware chain generation when a mux + // is registered as an inline group inside another mux. + inline bool + + // Set once any route has been registered through this mux (or, for an + // inline mux, through the parent it appends to). Used to reject Use() + // calls made after routes, whose middleware would otherwise be dropped. + hasRoutes bool + + // constraints contains inherited and locally registered parameter matchers. + constraints map[string]string + + // localConstraints distinguishes a duplicate registration in this scope + // from an intentional override of an inherited constraint. + localConstraints map[string]struct{} + + routes routes +} + +type routes struct { + rts []route +} + +func (r *routes) append(rt route) { + r.rts = append(r.rts, rt) +} + +type route struct { + regex *regexp.Regexp + methodhandler map[string]http.Handler + varNames []string + template string + remainder int +} + +// Logger is the minimal logging surface mux uses. *slog.Logger +// satisfies it directly, so New(WithLogger(slog.Default())) works without an +// adapter; other loggers need only a small shim. +type Logger interface { + Debug(msg string, args ...any) +} + +// sanitizeLogInput replaces newline characters to prevent log injection +func sanitizeLogInput(input string) string { + escaped := strings.NewReplacer("\n", "\\n", "\r", "\\r") + return escaped.Replace(input) +} + +// noopLogger is the default logger: a library should not write to the global +// logger unless the caller asks it to. +type noopLogger struct{} + +func (noopLogger) Debug(string, ...any) {} + +// Option configures a Mux at construction time. Pass options to New. +type Option func(*Mux) + +// WithNotFoundHandler sets the handler invoked when no route matches the +// request path. +func WithNotFoundHandler(h http.HandlerFunc) Option { + return func(mx *Mux) { mx.notFoundHandler = h } +} + +// WithMethodNotAllowedHandler sets the handler invoked when a route matches the +// request path but not its method. +func WithMethodNotAllowedHandler(h http.HandlerFunc) Option { + return func(mx *Mux) { mx.methodNotAllowedHandler = h } +} + +// WithLogger sets the debug logger. By default the router logs nothing. +func WithLogger(l Logger) Option { + return func(mx *Mux) { mx.logger = l } +} + +// New returns a newly initialized Mux that implements the Router interface, +// configured by the given options. Call New() for defaults, or pass options +// such as WithNotFoundHandler to customize behavior. +func New(opts ...Option) *Mux { + mux := &Mux{ + constraints: make(map[string]string), + localConstraints: make(map[string]struct{}), + routes: routes{ + rts: []route{}, + }, + } + for _, opt := range opts { + opt(mux) + } + return mux +} + +// Use appends middleware to the router's middleware stack. +func (mx *Mux) Use(middlewares ...func(http.Handler) http.Handler) { + // Middleware chains are baked into each handler at registration time, so a + // middleware added after a route would silently never run. Fail loudly + // instead of dropping it. + if mx.hasRoutes { + panic("mux: all middlewares must be registered before routes") + } + mx.middlewares = append(mx.middlewares, middlewares...) +} + +// With returns an inline router using the supplied middleware. +func (mx *Mux) With(middlewares ...func(http.Handler) http.Handler) Router { + return &Mux{ + constraints: cloneConstraints(mx.constraints), + localConstraints: make(map[string]struct{}), + middlewares: middlewares, + parent: mx, + inline: true, + } +} + +// Group adds an inline router with its own middleware stack. +func (mx *Mux) Group(fn func(r Router)) Router { + im := mx.With() + if fn != nil { + fn(im) + } + return im +} + +// RegisterConstraint sets the regular-expression fragment used by parameters +// with name in routes registered through this router scope. Constraints are +// inherited by child scopes and may be overridden there. It panics when the +// name or expression is invalid, when the name was already registered in this +// scope, or when routes have already been registered through this scope. +func (mx *Mux) RegisterConstraint(name, expression string) { + if mx.hasRoutes { + panic("mux: all constraints must be registered before routes") + } + if err := validateConstraint(name, expression); err != nil { + panic("mux: " + err.Error()) + } + if _, ok := mx.localConstraints[name]; ok { + panic(fmt.Sprintf("mux: constraint %q is already registered", name)) + } + mx.constraints[name] = expression + mx.localConstraints[name] = struct{}{} +} + +// Route creates a child router and mounts it at pattern. Once pattern matches, +// the child router owns that path namespace, including its 404 and 405 +// responses. The matched prefix is removed from the path used for child route +// matching; handlers continue to see the original request URL. +func (mx *Mux) Route(pattern string, fn func(r Router)) Router { + if fn == nil { + panic(fmt.Sprintf("mux: Route requires a non-nil function for %q", pattern)) + } + compiled, err := compileTemplate(pattern, mx.constraints, true) + if err != nil { + panic(fmt.Sprintf("mux: invalid route prefix %q: %v", pattern, err)) + } + + sr := &Mux{ + constraints: cloneConstraints(mx.constraints), + localConstraints: make(map[string]struct{}), + parent: mx, + routes: routes{rts: []route{}}, + } + fn(sr) + + handler := mx.chainHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sr.ServeHTTP(w, r) + })) + for _, registered := range mx.destinationRoutes().rts { + if registered.remainder > 0 && registered.regex.String() == compiled.regex.String() { + panic(fmt.Sprintf("mux: route prefix %q is already mounted", pattern)) + } + } + mx.appendRoute(methodAll, pattern, compiled, handler) + return sr +} + +// Handle registers handler for all methods matching pattern. +func (mx *Mux) Handle(pattern string, handler http.Handler) { + mx.Method(methodAll, pattern, handler) +} + +// HandleFunc registers handler for all methods matching pattern. +func (mx *Mux) HandleFunc(pattern string, handler http.HandlerFunc) { + mx.Method(methodAll, pattern, handler) +} + +// Method registers handler for method and pattern. +func (mx *Mux) Method(method, pattern string, handler http.Handler) { + // Normalize the method so registrations are case-insensitive and match the + // upper-case r.Method values used at dispatch time. The wildcard sentinel + // is upper-case-stable, so this is safe for it too. + if method != methodAll { + method = strings.ToUpper(method) + } + handler = mx.chainHandler(handler) + mx.hasRoutes = true + compiled, err := compileTemplate(pattern, mx.constraints, false) + if err != nil { + panic(fmt.Sprintf("mux: invalid route template %q: %v", pattern, err)) + } + + for _, rr := range mx.routes.rts { + if rr.regex.String() == compiled.regex.String() && rr.remainder == 0 { + rr.methodhandler[method] = handler + return + } + } + mx.appendRoute(method, pattern, compiled, handler) +} + +func (mx *Mux) appendRoute(method, template string, compiled compiledTemplate, handler http.Handler) { + mx.hasRoutes = true + r := route{ + regex: compiled.regex, + methodhandler: map[string]http.Handler{method: handler}, + varNames: compiled.varNames, + template: template, + remainder: compiled.remainderIndex, + } + if destination := mx.destinationMux(); destination != mx { + destination.routes.append(r) + destination.hasRoutes = true + return + } + mx.routes.append(r) +} + +func (mx *Mux) destinationRoutes() *routes { + return &mx.destinationMux().routes +} + +func (mx *Mux) destinationMux() *Mux { + destination := mx + for destination.parent != nil && destination.inline { + destination = destination.parent + } + return destination +} + +// captureNames returns the names of a compiled pattern's capture groups (in +// order, excluding the whole-match group at index 0). Unnamed groups yield "". +func captureNames(re *regexp.Regexp) []string { + names := re.SubexpNames() + if len(names) <= 1 { + return nil + } + return names[1:] +} + +// MethodFunc registers handler for method and pattern. +func (mx *Mux) MethodFunc(method, pattern string, handler http.HandlerFunc) { + mx.Method(method, pattern, handler) +} + +// Connect registers a CONNECT handler for pattern. +func (mx *Mux) Connect(pattern string, handler http.HandlerFunc) { + mx.MethodFunc(http.MethodConnect, pattern, handler) +} + +// Delete registers a DELETE handler for pattern. +func (mx *Mux) Delete(pattern string, handler http.HandlerFunc) { + mx.MethodFunc(http.MethodDelete, pattern, handler) +} + +// Get registers a GET handler for pattern. +func (mx *Mux) Get(pattern string, handler http.HandlerFunc) { + mx.MethodFunc(http.MethodGet, pattern, handler) +} + +// Head registers a HEAD handler for pattern. +func (mx *Mux) Head(pattern string, handler http.HandlerFunc) { + mx.MethodFunc(http.MethodHead, pattern, handler) +} + +// Options registers an OPTIONS handler for pattern. +func (mx *Mux) Options(pattern string, handler http.HandlerFunc) { + mx.MethodFunc(http.MethodOptions, pattern, handler) +} + +// Patch registers a PATCH handler for pattern. +func (mx *Mux) Patch(pattern string, handler http.HandlerFunc) { + mx.MethodFunc(http.MethodPatch, pattern, handler) +} + +// Post registers a POST handler for pattern. +func (mx *Mux) Post(pattern string, handler http.HandlerFunc) { + mx.MethodFunc(http.MethodPost, pattern, handler) +} + +// Put registers a PUT handler for pattern. +func (mx *Mux) Put(pattern string, handler http.HandlerFunc) { + mx.MethodFunc(http.MethodPut, pattern, handler) +} + +// Trace registers a TRACE handler for pattern. +func (mx *Mux) Trace(pattern string, handler http.HandlerFunc) { + mx.MethodFunc(http.MethodTrace, pattern, handler) +} + +// NotFound sets the handler used when no route matches. +func (mx *Mux) NotFound(handler http.HandlerFunc) { + mx.notFoundHandler = handler +} + +// MethodNotAllowed sets the handler used when a path matches but its method does not. +func (mx *Mux) MethodNotAllowed(handler http.HandlerFunc) { + mx.methodNotAllowedHandler = handler +} + +func (mx *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + if requestpath, ok := r.Context().Value(ctxKeyRequestPath).(string); ok { + path = requestpath + } + + // pathMatched tracks whether any route matched the path but not the + // method, so we can distinguish 405 (Method Not Allowed) from 404 (Not + // Found) only after considering every overlapping pattern. + pathMatched := false + + for _, route := range mx.routes.rts { + matches := route.regex.FindStringSubmatch(path) + if len(matches) <= 0 { + continue + } + handler, ok := route.methodhandler[r.Method] + if !ok { + handler, ok = route.methodhandler[methodAll] + } + if !ok { + // This pattern matched the path but has no handler for the + // method. Keep scanning: another overlapping pattern may. + pathMatched = true + continue + } + + params, _ := r.Context().Value(ctxKeyParams).(map[string]string) + params1 := make(map[string]string, len(params)+len(matches)-1) + for name, value := range params { + params1[name] = value + } + for i, match := range matches[1:] { + if i > len(route.varNames)-1 || route.varNames[i] == "" { + // Unnamed capture group: not exposed as a parameter. + continue + } + params1[route.varNames[i]] = match + } + ctx := context.WithValue(r.Context(), ctxKeyParams, params1) + if route.remainder > 0 { + ctx = context.WithValue(ctx, ctxKeyRequestPath, matches[route.remainder]) + } + r.Pattern = appendPattern(r.Pattern, route.template) + handler.ServeHTTP(w, r.WithContext(ctx)) + return + } + + if pathMatched { + mx.handleMethodNotAllowed(w, r) + mx.log().Debug("method not allowed", "method", r.Method, "path", sanitizeLogInput(path)) + return + } + mx.handleNotFound(w, r) +} + +func appendPattern(parent, child string) string { + if parent == "" || parent == "/" { + return child + } + return parent + child +} + +// log resolves the logger for this mux: its own if set, otherwise the parent's, +// falling back to a no-op. This mirrors the NotFound/MethodNotAllowed fallback +// so sub-Routers inherit the logger configured on the root. +func (mx *Mux) log() Logger { + if mx.logger != nil { + return mx.logger + } + if mx.parent != nil { + return mx.parent.log() + } + return noopLogger{} +} + +func (mx *Mux) chainHandler(handler http.Handler) http.Handler { + for i := len(mx.middlewares) - 1; i >= 0; i-- { + handler = mx.middlewares[i](handler) + } + if mx.parent != nil && mx.inline { + handler = mx.parent.chainHandler(handler) + } + return handler +} + +func (mx *Mux) handleNotFound(w http.ResponseWriter, r *http.Request) { + if mx.notFoundHandler != nil { + mx.notFoundHandler(w, r) + return + } + if mx.parent != nil { + mx.parent.handleNotFound(w, r) + return + } + defaultNotFoundHandler(w, r) +} + +func defaultNotFoundHandler(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("not found")) +} + +func (mx *Mux) handleMethodNotAllowed(w http.ResponseWriter, r *http.Request) { + if mx.methodNotAllowedHandler != nil { + mx.methodNotAllowedHandler(w, r) + return + } + if mx.parent != nil { + mx.parent.handleMethodNotAllowed(w, r) + return + } + defaultMethodNotAllowedHandler(w, r) +} + +func defaultMethodNotAllowedHandler(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusMethodNotAllowed) + _, _ = w.Write([]byte("not allowed")) +} diff --git a/ociserver/mux/mux_test.go b/ociserver/mux/mux_test.go new file mode 100644 index 0000000..9089334 --- /dev/null +++ b/ociserver/mux/mux_test.go @@ -0,0 +1,326 @@ +package mux + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +type testCase struct { + name string + path string + method string + body io.Reader + expectedStatus int + expectedBody string +} + +type testContextKey string + +const middlewaresContextKey testContextKey = "middlewares" + +func TestMuxBasic(t *testing.T) { + m := New() + + m.Get(`/`, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte("ok")) + }) + m.Get(`/path`, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte("get path")) + }) + m.Post(`/path`, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte("post path")) + }) + m.Patch(`/path`, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte("patch path")) + }) + m.Get(`/*var1/:var2/path`, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + _, _ = fmt.Fprintf(w, "%s %s", URLParam(r, "var1"), URLParam(r, "var2")) + }) + m.HandleFunc(`/allmethods`, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte("all methods")) + }) + + ts := httptest.NewServer(m) + defer ts.Close() + + testCases := []testCase{ + { + name: "get root", + path: "/", + method: "GET", + expectedStatus: 200, + expectedBody: "ok", + }, + { + name: "not found", + path: "/notfound", + method: "GET", + expectedStatus: 404, + expectedBody: "not found", + }, { + name: "get path", + path: "/path", + method: "GET", + expectedStatus: 200, + expectedBody: "get path", + }, { + name: "post path", + path: "/path", + method: "POST", + expectedStatus: 200, + expectedBody: "post path", + }, { + name: "patch path", + path: "/path", + method: "PATCH", + expectedStatus: 200, + expectedBody: "patch path", + }, { + name: "delete path", + path: "/path", + method: "DELETE", + expectedStatus: 405, + expectedBody: "not allowed", + }, { + name: "get path with var1 and var2", + path: "/foo/bar/path", + method: "GET", + expectedStatus: 200, + expectedBody: "foo bar", + }, { + name: "all methods", + path: "/allmethods", + method: "OPTIONS", + expectedStatus: 200, + expectedBody: "all methods", + }, + } + + runTestCases(t, ts, testCases) +} + +func TestGrouping(t *testing.T) { + m := New() + + m.Use(func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + v, ok := r.Context().Value(middlewaresContextKey).([]string) + if !ok { + v = []string{} + } + v = append(v, "1") + r = r.WithContext(context.WithValue(r.Context(), middlewaresContextKey, v)) + next.ServeHTTP(w, r) + }) + }) + + m.Group(func(r Router) { + r.Use(func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + v, ok := r.Context().Value(middlewaresContextKey).([]string) + if !ok { + t.Fatalf("failed to get middlewares from context") + } + v = append(v, "a") + r = r.WithContext(context.WithValue(r.Context(), middlewaresContextKey, v)) + next.ServeHTTP(w, r) + }) + }) + r.Get(`/foo`, returnMWs(t)) + }) + + m.Group(func(r Router) { + r.Use(func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + v, ok := r.Context().Value(middlewaresContextKey).([]string) + if !ok { + t.Fatalf("failed to get middlewares from context") + } + v = append(v, "b") + r = r.WithContext(context.WithValue(r.Context(), middlewaresContextKey, v)) + next.ServeHTTP(w, r) + }) + }) + r.Get(`/bar`, returnMWs(t)) + }) + m.Get(`/`, returnMWs(t)) + ts := httptest.NewServer(m) + defer ts.Close() + + testCases := []testCase{ + { + name: "get root", + path: "/", + method: "GET", + expectedStatus: 200, + expectedBody: "1", + }, { + name: "get foo", + path: "/foo", + method: "GET", + expectedStatus: 200, + expectedBody: "1 a", + }, { + name: "get bar", + path: "/bar", + method: "GET", + expectedStatus: 200, + expectedBody: "1 b", + }, + } + + runTestCases(t, ts, testCases) +} + +func TestOCIDistRouting(t *testing.T) { + m := New() + + m.Route("/v2", func(r Router) { + r.Head("/*name/manifests/:reference", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + }) + r.Get("/*name/manifests/:reference", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + fmt.Fprintf(w, "manifest get: %s %s", URLParam(r, "name"), URLParam(r, "reference")) + }) + r.Put("/*name/manifests/:reference", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + fmt.Fprintf(w, "manifest put: %s %s", URLParam(r, "name"), URLParam(r, "reference")) + }) + r.Delete("/*name/manifests/:reference", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + fmt.Fprintf(w, "manifest delete: %s %s", URLParam(r, "name"), URLParam(r, "reference")) + }) + }) + + ts := httptest.NewServer(m) + defer ts.Close() + + testCases := []testCase{ + { + name: "head", + path: "/v2/foo/bar/baz/manifests/tag", + method: "HEAD", + expectedStatus: 200, + expectedBody: "", + }, { + name: "get", + path: "/v2/foo/manifests/tag", + method: "GET", + expectedStatus: 200, + expectedBody: "manifest get: foo tag", + }, { + name: "put", + path: "/v2/foo/bar/manifests/tag", + method: "PUT", + expectedStatus: 200, + expectedBody: "manifest put: foo/bar tag", + }, { + name: "delete", + path: "/v2/foo/bar/baz/manifests/tag", + method: "DELETE", + expectedStatus: 200, + expectedBody: "manifest delete: foo/bar/baz tag", + }, + } + + runTestCases(t, ts, testCases) +} + +// TestMethodDispatchAcrossOverlappingPatterns guards against the 405 +// short-circuit: a request whose method is served by a later, overlapping +// pattern must be dispatched rather than rejected by the first pattern that +// only matched the path. +func TestMethodDispatchAcrossOverlappingPatterns(t *testing.T) { + m := New() + m.Get(`/:value`, func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("get")) + }) + m.Post(`/x`, func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("post")) + }) + + ts := httptest.NewServer(m) + defer ts.Close() + + testCases := []testCase{ + { + name: "get matches first pattern", + path: "/x", + method: http.MethodGet, + expectedStatus: http.StatusOK, + expectedBody: "get", + }, { + name: "post falls through to overlapping pattern", + path: "/x", + method: http.MethodPost, + expectedStatus: http.StatusOK, + expectedBody: "post", + }, { + name: "unhandled method still yields 405", + path: "/x", + method: http.MethodDelete, + expectedStatus: http.StatusMethodNotAllowed, + expectedBody: "not allowed", + }, + } + + runTestCases(t, ts, testCases) +} + +func returnMWs(t *testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + v, ok := r.Context().Value(middlewaresContextKey).([]string) + if !ok { + t.Fatalf("failed to get middlewares from context") + } + w.WriteHeader(200) + w.Write([]byte(strings.Join(v, " "))) + } +} + +func runTestCases(t *testing.T, ts *httptest.Server, testCases []testCase) { + for _, tc := range testCases { + resp, body := testRequest(t, ts, tc.method, tc.path, tc.body) + if resp.StatusCode != tc.expectedStatus { + t.Fatalf("test case '%s' failed, expected status %d, got %d", tc.name, tc.expectedStatus, resp.StatusCode) + } + if body != tc.expectedBody { + t.Fatalf("test case '%s' failed, expected body '%s', got '%s'", tc.name, tc.expectedBody, body) + } + } +} + +func testRequest(t *testing.T, ts *httptest.Server, method, path string, body io.Reader) (*http.Response, string) { + req, err := http.NewRequest(method, ts.URL+path, body) + if err != nil { + t.Fatal(err) + return nil, "" + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + return nil, "" + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + return nil, "" + } + defer resp.Body.Close() + + return resp, string(respBody) +} diff --git a/ociserver/mux/router.go b/ociserver/mux/router.go new file mode 100644 index 0000000..c821a4a --- /dev/null +++ b/ociserver/mux/router.go @@ -0,0 +1,86 @@ +// Package mux routes HTTP requests using path templates compiled to Go +// regular expressions. +// +// # Templates +// +// Templates match the complete path visible to their router and are anchored +// automatically. Literal segments are escaped. A :placeholder matches one +// non-empty segment, while a *wildcard matches one or more segments: +// +// m.Get("/users/:id", func(w http.ResponseWriter, r *http.Request) { +// id := mux.URLParam(r, "id") +// // ... +// }) +// +// m.Get("/files/*path", func(w http.ResponseWriter, r *http.Request) { +// path := mux.URLParam(r, "path") +// // ... +// }) +// +// A placeholder or wildcard must occupy its complete path segment. Use +// RegisterConstraint before registering routes to replace the default matcher +// for parameters with a particular name. +// +// Routes are evaluated in registration order and the first matching route with +// a handler for the request method wins. +// +// # Sub-routers +// +// Route mounts a child router at a path prefix. The prefix is removed from the +// path used for matching in the child, while handlers retain the original URL: +// +// m.Route("/api", func(r mux.Router) { +// r.Get("/widgets/:id", ...) // matches GET /api/widgets/123 +// }) +// +// Once a Route prefix matches, its child owns the namespace and produces the +// final 404 or 405 response when no child endpoint handles the request. +package mux + +import "net/http" + +// Router is the routing surface implemented by Mux. +type Router interface { + http.Handler + + // Use appends one or more middlewares to the Router stack. + Use(middlewares ...func(http.Handler) http.Handler) + + // Group adds an inline Router with a fresh middleware and constraint scope. + Group(fn func(r Router)) Router + + // Route creates and mounts a child Router at pattern. + Route(pattern string, fn func(r Router)) Router + + // RegisterConstraint defines the regular-expression fragment used for + // parameters with name in routes registered through this scope. + RegisterConstraint(name, expression string) + + // Handle and HandleFunc add routes matching all HTTP methods. + Handle(pattern string, h http.Handler) + HandleFunc(pattern string, h http.HandlerFunc) + + // Method and MethodFunc add routes matching method. + Method(method, pattern string, h http.Handler) + MethodFunc(method, pattern string, h http.HandlerFunc) + + Connect(pattern string, h http.HandlerFunc) + Delete(pattern string, h http.HandlerFunc) + Get(pattern string, h http.HandlerFunc) + Head(pattern string, h http.HandlerFunc) + Options(pattern string, h http.HandlerFunc) + Patch(pattern string, h http.HandlerFunc) + Post(pattern string, h http.HandlerFunc) + Put(pattern string, h http.HandlerFunc) + Trace(pattern string, h http.HandlerFunc) + + // NotFound defines a handler for unmatched routes. + NotFound(h http.HandlerFunc) + + // MethodNotAllowed defines a handler for matched paths without a handler for + // the request method. + MethodNotAllowed(h http.HandlerFunc) +} + +// Middlewares is a slice of standard middleware handlers. +type Middlewares []func(http.Handler) http.Handler diff --git a/ociserver/mux/template.go b/ociserver/mux/template.go new file mode 100644 index 0000000..6c5c60a --- /dev/null +++ b/ociserver/mux/template.go @@ -0,0 +1,176 @@ +package mux + +import ( + "fmt" + "maps" + "regexp" + "regexp/syntax" + "slices" + "strings" +) + +const ( + defaultPlaceholderExpression = `[^/]+` + defaultWildcardExpression = `.+` +) + +type compiledTemplate struct { + regex *regexp.Regexp + varNames []string + remainderIndex int +} + +// ValidPattern reports whether pattern is a valid route template. Route +// templates begin with '/', contain literal path segments, and may use +// :placeholder or *wildcard segments. +func ValidPattern(pattern string) error { + _, err := compileTemplate(pattern, nil, false) + return err +} + +func compileTemplate(pattern string, constraints map[string]string, prefix bool) (compiledTemplate, error) { + if pattern == "" || pattern[0] != '/' { + return compiledTemplate{}, fmt.Errorf("route template must begin with '/'") + } + if prefix && pattern != "/" && strings.HasSuffix(pattern, "/") { + return compiledTemplate{}, fmt.Errorf("route prefix must not end with '/'") + } + if prefix && pattern == "/" { + re := regexp.MustCompile(`^(/.*)$`) + return compiledTemplate{ + regex: re, + varNames: captureNames(re), + remainderIndex: 1, + }, nil + } + + var expression strings.Builder + expression.WriteByte('^') + + seen := make(map[string]struct{}) + segments := strings.Split(pattern, "/") + for i, segment := range segments { + if i > 0 { + expression.WriteByte('/') + } + if segment == "" { + continue + } + + marker := segment[0] + if marker != ':' && marker != '*' { + expression.WriteString(regexp.QuoteMeta(segment)) + continue + } + + name := segment[1:] + if !validIdentifier(name) { + return compiledTemplate{}, fmt.Errorf("invalid parameter %q", segment) + } + if _, ok := seen[name]; ok { + return compiledTemplate{}, fmt.Errorf("parameter %q appears more than once", name) + } + seen[name] = struct{}{} + + if prefix && marker == '*' { + return compiledTemplate{}, fmt.Errorf("wildcard parameter %q is not allowed in a route prefix", name) + } + + matcher := defaultPlaceholderExpression + if marker == '*' { + matcher = defaultWildcardExpression + } + if constraint, ok := constraints[name]; ok { + matcher = constraint + } + expression.WriteString("(?P<") + expression.WriteString(name) + expression.WriteString(">(?:") + expression.WriteString(matcher) + expression.WriteString("))") + } + + remainderIndex := 0 + if prefix { + // A mounted prefix owns both its exact path and everything below it. + // The optional capture preserves the leading slash for the child router. + expression.WriteString(`(?:(/.*))?$`) + } else { + expression.WriteByte('$') + } + + re, err := regexp.Compile(expression.String()) + if err != nil { + return compiledTemplate{}, err + } + if prefix { + remainderIndex = re.NumSubexp() + } + return compiledTemplate{ + regex: re, + varNames: captureNames(re), + remainderIndex: remainderIndex, + }, nil +} + +func validIdentifier(name string) bool { + for i, r := range name { + if i == 0 { + if !isASCIILetter(r) { + return false + } + continue + } + if !isASCIILetter(r) && (r < '0' || r > '9') && r != '_' { + return false + } + } + return name != "" +} + +func isASCIILetter(r rune) bool { + return r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' +} + +func validateConstraint(name, expression string) error { + if !validIdentifier(name) { + return fmt.Errorf("invalid constraint name %q", name) + } + if expression == "" { + return fmt.Errorf("constraint %q has an empty expression", name) + } + + re, err := regexp.Compile(expression) + if err != nil { + return fmt.Errorf("constraint %q: %w", name, err) + } + if re.NumSubexp() != 0 { + return fmt.Errorf("constraint %q must not contain capturing groups", name) + } + if re.MatchString("") { + return fmt.Errorf("constraint %q must not match an empty string", name) + } + + parsed, err := syntax.Parse(expression, syntax.Perl) + if err != nil { + return fmt.Errorf("constraint %q: %w", name, err) + } + if containsAnchor(parsed) { + return fmt.Errorf("constraint %q must not contain anchors", name) + } + return nil +} + +func containsAnchor(re *syntax.Regexp) bool { + switch re.Op { + case syntax.OpBeginLine, syntax.OpEndLine, syntax.OpBeginText, syntax.OpEndText: + return true + } + return slices.ContainsFunc(re.Sub, containsAnchor) +} + +func cloneConstraints(src map[string]string) map[string]string { + dst := make(map[string]string, len(src)) + maps.Copy(dst, src) + return dst +} diff --git a/ociserver/mux/template_test.go b/ociserver/mux/template_test.go new file mode 100644 index 0000000..2fdb61a --- /dev/null +++ b/ociserver/mux/template_test.go @@ -0,0 +1,201 @@ +package mux + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +func TestTemplateParametersAndLiterals(t *testing.T) { + m := New() + m.Get("/literal.+/:id/*rest", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, "%s|%s|%s", URLParam(r, "id"), URLParam(r, "rest"), r.Pattern) + }) + + rec := httptest.NewRecorder() + m.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/literal.+/42/a/b", nil)) + if got, want := rec.Body.String(), "42|a/b|/literal.+/:id/*rest"; got != want { + t.Fatalf("body = %q, want %q", got, want) + } + + rec = httptest.NewRecorder() + m.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/literalZZ/42/a/b", nil)) + if rec.Code != http.StatusNotFound { + t.Fatalf("literal metacharacters were not escaped: status = %d", rec.Code) + } +} + +func TestConstraintsAreScopedAndInherited(t *testing.T) { + m := New() + m.RegisterConstraint("id", `[0-9]+`) + m.Get("/numbers/:id", echoParam("id")) + + m.Group(func(r Router) { + r.RegisterConstraint("id", `[a-z]+`) + r.Get("/letters/:id", echoParam("id")) + }) + + m.Route("/api", func(r Router) { + r.Get("/items/:id", echoParam("id")) + }) + + tests := []struct { + path string + status int + body string + }{ + {path: "/numbers/123", status: http.StatusOK, body: "123"}, + {path: "/numbers/abc", status: http.StatusNotFound, body: "not found"}, + {path: "/letters/abc", status: http.StatusOK, body: "abc"}, + {path: "/letters/123", status: http.StatusNotFound, body: "not found"}, + {path: "/api/items/123", status: http.StatusOK, body: "123"}, + {path: "/api/items/abc", status: http.StatusNotFound, body: "not found"}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + rec := httptest.NewRecorder() + m.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tt.path, nil)) + if rec.Code != tt.status || rec.Body.String() != tt.body { + t.Fatalf("response = (%d, %q), want (%d, %q)", rec.Code, rec.Body.String(), tt.status, tt.body) + } + }) + } +} + +func TestRouteStripsPrefixAndRetainsOriginalRequest(t *testing.T) { + m := New() + m.Route("/api/:version", func(r Router) { + r.Use(func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, _ = fmt.Fprintf(w, "middleware=%s;", URLParam(req, "id")) + next.ServeHTTP(w, req) + }) + }) + r.Get("/items/:id", func(w http.ResponseWriter, req *http.Request) { + _, _ = fmt.Fprintf(w, "version=%s;id=%s;path=%s;pattern=%s", + URLParam(req, "version"), URLParam(req, "id"), req.URL.Path, req.Pattern) + }) + }) + + rec := httptest.NewRecorder() + m.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v2/items/42", nil)) + want := "middleware=42;version=v2;id=42;path=/api/v2/items/42;pattern=/api/:version/items/:id" + if got := rec.Body.String(); got != want { + t.Fatalf("body = %q, want %q", got, want) + } +} + +func TestNestedRoute(t *testing.T) { + m := New() + m.Route("/api", func(r Router) { + r.Route("/v1", func(r Router) { + r.Get("/widgets/:id", func(w http.ResponseWriter, req *http.Request) { + _, _ = fmt.Fprintf(w, "%s %s", URLParam(req, "id"), req.Pattern) + }) + }) + }) + + rec := httptest.NewRecorder() + m.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/widgets/7", nil)) + if got, want := rec.Body.String(), "7 /api/v1/widgets/:id"; got != want { + t.Fatalf("body = %q, want %q", got, want) + } +} + +func TestRouteOwnsMatchedPrefix(t *testing.T) { + m := New() + m.Route("/api", func(r Router) { + r.Get("/known", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("known")) + }) + r.NotFound(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusTeapot) + _, _ = w.Write([]byte("child not found")) + }) + }) + m.Get("/api/shadowed", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("root")) + }) + + rec := httptest.NewRecorder() + m.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/shadowed", nil)) + if rec.Code != http.StatusTeapot || rec.Body.String() != "child not found" { + t.Fatalf("response = (%d, %q), want child not found", rec.Code, rec.Body.String()) + } +} + +func TestRouteRejectsDuplicateMount(t *testing.T) { + m := New() + m.Route("/api", func(Router) {}) + assertPanics(t, func() { m.Route("/api", func(Router) {}) }) + assertPanics(t, func() { m.Route("/nil", nil) }) + assertPanics(t, func() { m.Route("/trailing/", func(Router) {}) }) +} + +func TestRouteAtRoot(t *testing.T) { + m := New() + m.Route("/", func(r Router) { + r.Get("/items/:id", func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, "%s %s", URLParam(r, "id"), r.Pattern) + }) + }) + + rec := httptest.NewRecorder() + m.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/items/9", nil)) + if rec.Code != http.StatusOK || rec.Body.String() != "9 /items/:id" { + t.Fatalf("response = (%d, %q), want (200, %q)", rec.Code, rec.Body.String(), "9 /items/:id") + } +} + +func TestValidPattern(t *testing.T) { + for _, pattern := range []string{"/", "/users/:id", "/files/*path", "/literal.+"} { + if err := ValidPattern(pattern); err != nil { + t.Errorf("ValidPattern(%q) = %v", pattern, err) + } + } + for _, pattern := range []string{"", "users/:id", "/users/:", "/:id/:id", "/files/*"} { + if err := ValidPattern(pattern); err == nil { + t.Errorf("ValidPattern(%q) unexpectedly succeeded", pattern) + } + } +} + +func TestRegisterConstraintPanics(t *testing.T) { + tests := []func(*Mux){ + func(m *Mux) { m.RegisterConstraint("bad-name", `[a-z]+`) }, + func(m *Mux) { m.RegisterConstraint("id", ``) }, + func(m *Mux) { m.RegisterConstraint("id", `([a-z]+)`) }, + func(m *Mux) { m.RegisterConstraint("id", `^[a-z]+$`) }, + func(m *Mux) { m.RegisterConstraint("id", `[a-z]*`) }, + func(m *Mux) { + m.RegisterConstraint("id", `[a-z]+`) + m.RegisterConstraint("id", `[0-9]+`) + }, + func(m *Mux) { + m.Get("/items/:id", echoParam("id")) + m.RegisterConstraint("id", `[0-9]+`) + }, + } + for i, test := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + assertPanics(t, func() { test(New()) }) + }) + } +} + +func echoParam(name string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(URLParam(r, name))) + } +} + +func assertPanics(t *testing.T, fn func()) { + t.Helper() + defer func() { + if recover() == nil { + t.Fatal("function did not panic") + } + }() + fn() +} diff --git a/ociserver/proxy_test.go b/ociserver/proxy_test.go deleted file mode 100644 index 70f8ec8..0000000 --- a/ociserver/proxy_test.go +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright 2023 CUE Labs AG -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ociserver_test - -import ( - "bytes" - "context" - "fmt" - "net/http" - "net/http/httptest" - "testing" - - "github.com/docker/oci" - "github.com/docker/oci/ocidigest" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/docker/oci/ociclient" - "github.com/docker/oci/ocimem" - "github.com/docker/oci/ociserver" -) - -// Test that implementing an OCI registry proxy by sitting ociserver -// in front of ociclient doesn't introduce extra HTTP requests to the proxy backend. -// -// Each test case begins with a backend registry (ociserver in front of ocimem) -// with an HTTP middleware to record backend requests as they come in. -// -// We then set up a proxy (ociserver in front of ociclient) where the client points at the backend, -// and the server has a similar middleware to record the proxy requests as they come in. -// -// Finally, we have an ociclient pointing at the proxy which performs an OCI action via clientDo. -// We expect the proxy and backend requests to be practically the same -// as long as ociserver and ociclient do the right thing. - -// ociclient defaults to a chunk size of 64KiB. -// We want our small data to fit in a single chunk, -// and large data to need at least three chunks to properly test PATCH edge cases. -var ( - smallData = bytes.Repeat([]byte("x"), 10) // 10 B - largeData = bytes.Repeat([]byte("x"), 150*1024) // 150 KiB -) - -var proxyTests = []struct { - name string - clientDo func(context.Context, oci.Interface) error - - proxyRequests []string - backendRequests []string -}{ - { - name: "PushBlob_small", - clientDo: func(ctx context.Context, client oci.Interface) error { - _, err := client.PushBlob(ctx, "foo/bar", oci.Descriptor{ - Size: int64(len(smallData)), - Digest: ocidigest.FromBytes(smallData), - }, bytes.NewReader(smallData)) - return err - }, - proxyRequests: []string{ - "POST len=0", - "PUT len=10", - }, - backendRequests: []string{ - "POST len=0", - "PUT len=10", - }, - }, - { - name: "PushBlob_large", - clientDo: func(ctx context.Context, client oci.Interface) error { - _, err := client.PushBlob(ctx, "foo/bar", oci.Descriptor{ - Size: int64(len(largeData)), - Digest: ocidigest.FromBytes(largeData), - }, bytes.NewReader(largeData)) - return err - }, - proxyRequests: []string{ - "POST len=0", - "PUT len=153600", - }, - backendRequests: []string{ - "POST len=0", - "PUT len=153600", - }, - }, - { - name: "PushBlobChunked_large_oneWrite", - clientDo: func(ctx context.Context, client oci.Interface) error { - bw, err := client.PushBlobChunked(ctx, "foo/bar", 0) - if err != nil { - return err - } - if _, err := bw.Write(largeData); err != nil { - return err - } - if _, err := bw.Commit(ocidigest.FromBytes(largeData)); err != nil { - return err - } - return nil - }, - proxyRequests: []string{ - "POST len=0", - "PATCH len=153600", - "PUT len=0", - }, - backendRequests: []string{ - "POST len=0", - "PATCH len=153600", - "PUT len=0", - }, - }, -} - -func recordingServer(tb testing.TB, reqs *[]string, handler http.Handler) *httptest.Server { - recHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - *reqs = append(*reqs, fmt.Sprintf("%s len=%d", r.Method, r.ContentLength)) - handler.ServeHTTP(w, r) - }) - server := httptest.NewServer(recHandler) - tb.Cleanup(server.Close) - return server -} - -func testClient(tb testing.TB, server *httptest.Server) oci.Interface { - client, err := ociclient.New(server.Listener.Addr().String(), &ociclient.Options{ - Insecure: true, // since it's a local httptest server - }) - require.NoError(tb, err) - return client -} - -func TestProxyRequests(t *testing.T) { - for _, test := range proxyTests { - t.Run(test.name, func(t *testing.T) { - // Set up the backend (ociserver + ocimem) - var proxyReqs, backendReqs []string - backendServer := recordingServer(t, &backendReqs, - ociserver.New(ocimem.New(), nil)) - - // Set up the proxy (ociserver + ociclient). - proxyServer := recordingServer(t, &proxyReqs, - ociserver.New(testClient(t, backendServer), nil)) - - // Set up the input client, mimicking the end user like cmd/cue. - inputClient := testClient(t, proxyServer) - - // Run the input client action, and compare the results. - err := test.clientDo(context.TODO(), inputClient) - require.NoError(t, err) - - assert.Equal(t, test.proxyRequests, proxyReqs) - assert.Equal(t, test.backendRequests, backendReqs) - }) - } -} diff --git a/ociserver/range.go b/ociserver/range.go deleted file mode 100644 index 5e6e1c6..0000000 --- a/ociserver/range.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2023 CUE Labs AG -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ociserver - -import ( - "errors" - "net/textproto" - "strconv" - "strings" -) - -// This adapted from the code in fs.go in the net/http package. -// The main difference is that it's more limited because we -// don't have access to the size before parsing the range, -// (because otherwise we'd have to make an extra round trip -// to fetch the size before making the actual request). - -// httpRange specifies a byte range as requested by a client. -// If end is negative, it represents the end of the file. -type httpRange struct { - start, end int64 -} - -// parseRange parses a Range header string as per RFC 7233. -func parseRange(s string) ([]httpRange, error) { - if s == "" { - return nil, nil // header not present - } - const b = "bytes=" - if !strings.HasPrefix(s, b) { - return nil, errors.New("invalid range") - } - ranges := make([]httpRange, 0, 1) - for _, ra := range strings.Split(s[len(b):], ",") { - ra = textproto.TrimString(ra) - if ra == "" { - continue - } - start, end, ok := strings.Cut(ra, "-") - if !ok { - return nil, errors.New("invalid range") - } - start, end = textproto.TrimString(start), textproto.TrimString(end) - var r httpRange - if start == "" { - // If no start is specified, end specifies the - // range start relative to the end of the file, - // and we are dealing with - // which has to be a non-negative integer as per - // RFC 7233 Section 2.1 "Byte-Ranges". - if end == "" || end[0] == '-' { - return nil, errors.New("invalid range") - } - return nil, errors.New("end-relative range not supported") - } else { - i, err := strconv.ParseInt(start, 10, 64) - if err != nil || i < 0 { - return nil, errors.New("invalid range") - } - r.start = i - if end == "" { - // If no end is specified, range extends to end of the file. - r.end = -1 - } else { - i, err := strconv.ParseInt(end, 10, 64) - if err != nil || r.start > i { - return nil, errors.New("invalid range") - } - r.end = i + 1 - } - } - ranges = append(ranges, r) - } - return ranges, nil -} diff --git a/ociserver/reader.go b/ociserver/reader.go deleted file mode 100644 index d3aba8f..0000000 --- a/ociserver/reader.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ociserver - -import ( - "context" - "fmt" - "io" - "net/http" - - "github.com/docker/oci" - "github.com/docker/oci/internal/ocirequest" - "github.com/docker/oci/ocidigest" -) - -func (r *registry) handleBlobHead(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - digest, err := ocidigest.Parse(rreq.Digest) - if err != nil { - return err - } - desc, err := r.backend.ResolveBlob(ctx, rreq.Repo, digest) - if err != nil { - return err - } - resp.Header().Set("Content-Length", fmt.Sprint(desc.Size)) - resp.Header().Set("Docker-Content-Digest", desc.Digest.String()) - // TODO this is true in theory, but what if the backend doesn't support GetBlobRange ? - resp.Header().Set("Accept-Ranges", "bytes") - resp.WriteHeader(http.StatusOK) - return nil -} - -func (r *registry) handleBlobGet(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - if r.opts.LocationsForDescriptor != nil { - // We need to find information on the blob before we can determine - // what to pass back, so resolve the blob first so we don't - // stimulate the backend to start sending the whole stream - // only to abandon it. - digest, err := ocidigest.Parse(rreq.Digest) - if err != nil { - return err - } - desc, err := r.backend.ResolveBlob(ctx, rreq.Repo, digest) - if err != nil { - // TODO this might not be the best response because ResolveBlob is - // often implemented with a HEAD request that can't return an error - // body. So it might be better to fall through to the usual GetBlob request, - // although that would mean that every error makes two calls :( - return err - } - locs, err := r.opts.LocationsForDescriptor(false, desc) - if err != nil { - return err - } - if len(locs) > 0 { - // TODO choose randomly from the set of locations? - // TODO make it possible to turn off this behaviour? - http.Redirect(resp, req, locs[0], http.StatusTemporaryRedirect) - return nil - } - } - ranges, err := parseRange(req.Header.Get("Range")) - if err != nil { - return withHTTPCode(http.StatusRequestedRangeNotSatisfiable, err) - } - switch len(ranges) { - case 0: - digest, err := ocidigest.Parse(rreq.Digest) - if err != nil { - return err - } - blob, err := r.backend.GetBlob(ctx, rreq.Repo, digest) - if err != nil { - return err - } - defer blob.Close() - desc := blob.Descriptor() - resp.Header().Set("Content-Type", desc.MediaType) - resp.Header().Set("Content-Length", fmt.Sprint(desc.Size)) - resp.Header().Set("Docker-Content-Digest", rreq.Digest) - resp.WriteHeader(http.StatusOK) - - io.Copy(resp, blob) - return nil - case 1: - rng := ranges[0] - digest, err := ocidigest.Parse(rreq.Digest) - if err != nil { - return err - } - blob, err := r.backend.GetBlobRange(ctx, rreq.Repo, digest, rng.start, rng.end) - if err != nil { - // TODO fall back to using GetBlob if err is ErrUnsupported? - return err - } - defer blob.Close() - desc := blob.Descriptor() - if rng.end == -1 || rng.end > desc.Size { - rng.end = desc.Size - } - if rng.start > desc.Size { - return withHTTPCode(http.StatusRequestedRangeNotSatisfiable, fmt.Errorf("range starts after end of blob")) - } - if rng.end < rng.start { - return withHTTPCode(http.StatusRequestedRangeNotSatisfiable, fmt.Errorf("range end is before start")) - } - resp.Header().Set("Content-Type", desc.MediaType) - resp.Header().Set("Content-Length", fmt.Sprint(rng.end-rng.start)) - resp.Header().Set("Docker-Content-Digest", rreq.Digest) - resp.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", rng.start, rng.end-1, desc.Size)) - resp.WriteHeader(http.StatusPartialContent) - - io.Copy(resp, blob) - return nil - - default: - return withHTTPCode(http.StatusRequestedRangeNotSatisfiable, fmt.Errorf("only a single range is supported")) - } -} - -func (r *registry) handleManifestGet(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - // TODO we could do a redirect here too if we thought it was worthwhile. - var mr oci.BlobReader - var err error - if rreq.Tag != "" { - mr, err = r.backend.GetTag(ctx, rreq.Repo, rreq.Tag) - } else { - digest, parseErr := ocidigest.Parse(rreq.Digest) - if parseErr != nil { - return parseErr - } - mr, err = r.backend.GetManifest(ctx, rreq.Repo, digest) - } - if err != nil { - return err - } - if mr == nil { - return fmt.Errorf("backend returned nil manifest reader") - } - desc := mr.Descriptor() - if !r.opts.OmitDigestFromTagGetResponse { - resp.Header().Set("Docker-Content-Digest", desc.Digest.String()) - } - resp.Header().Set("Content-Type", desc.MediaType) - resp.Header().Set("Content-Length", fmt.Sprint(desc.Size)) - resp.WriteHeader(http.StatusOK) - io.Copy(resp, mr) - return nil -} - -func (r *registry) handleManifestHead(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - var desc oci.Descriptor - var err error - if rreq.Tag != "" { - desc, err = r.backend.ResolveTag(ctx, rreq.Repo, rreq.Tag) - } else { - digest, parseErr := ocidigest.Parse(rreq.Digest) - if parseErr != nil { - return parseErr - } - desc, err = r.backend.ResolveManifest(ctx, rreq.Repo, digest) - } - if err != nil { - return err - } - if !r.opts.OmitDigestFromTagGetResponse || rreq.Tag != "" { - // Note: when doing a HEAD of a tag, clients are entitled - // to expect that the digest header is set on the response - // even though the spec says it's only optional in this case. - // TODO raise an issue on the spec about this. - resp.Header().Set("Docker-Content-Digest", desc.Digest.String()) - } - resp.Header().Set("Content-Type", desc.MediaType) - resp.Header().Set("Content-Length", fmt.Sprint(desc.Size)) - resp.WriteHeader(http.StatusOK) - return nil -} diff --git a/ociserver/referrers.go b/ociserver/referrers.go new file mode 100644 index 0000000..6ef5560 --- /dev/null +++ b/ociserver/referrers.go @@ -0,0 +1,73 @@ +package ociserver + +import ( + "encoding/json" + "errors" + "net/http" + + "github.com/docker/oci" + "github.com/docker/oci/ocidigest" + "github.com/docker/oci/ociserver/mux" +) + +func marshalReferrersResponse(descs []oci.Descriptor) ([]byte, error) { + return json.Marshal(map[string]any{ + "schemaVersion": 2, + "mediaType": oci.MediaTypeImageIndex, + "manifests": descs, + }) +} + +func (s *Server) referrersGet() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := mux.URLParam(r, "name") + dgstString := mux.URLParam(r, "digest") + artifactType := r.URL.Query().Get("artifactType") + + dgst, err := ocidigest.Parse(dgstString) + if err != nil { + returnError(w, ErrManifestInvalid("invalid digest")) + return + } + descSeq := s.db.Referrers(r.Context(), name, dgst, &oci.ReferrersParameters{ArtifactType: artifactType}) + descs, err := oci.All(descSeq) + if err != nil { + if errors.Is(err, oci.ErrManifestUnknown) || errors.Is(err, oci.ErrNameUnknown) { + body, encErr := marshalReferrersResponse([]oci.Descriptor{}) + if encErr != nil { + s.logError(r.Context(), "encoding empty referrers response", encErr, "repository", name, "digest", dgst, "artifactType", artifactType) + returnError(w, ErrServerError()) + return + } + w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json") + if artifactType != "" { + w.Header().Set("OCI-Filters-Applied", "artifactType") + } + if _, err := w.Write(body); err != nil { + s.logError(r.Context(), "writing empty referrers response", err, "repository", name, "digest", dgst, "artifactType", artifactType) + } + return + } + s.logError(r.Context(), "listing referrers", err, "repository", name, "digest", dgst, "artifactType", artifactType) + returnError(w, ErrServerError()) + return + } + + if descs == nil { + descs = []oci.Descriptor{} + } + body, err := marshalReferrersResponse(descs) + if err != nil { + s.logError(r.Context(), "encoding referrers response", err, "repository", name, "digest", dgst, "artifactType", artifactType) + returnError(w, ErrServerError()) + return + } + w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json") + if artifactType != "" { + w.Header().Set("OCI-Filters-Applied", "artifactType") + } + if _, err := w.Write(body); err != nil { + s.logError(r.Context(), "writing referrers response", err, "repository", name, "digest", dgst, "artifactType", artifactType) + } + } +} diff --git a/ociserver/registry.go b/ociserver/registry.go deleted file mode 100644 index 8e48cc7..0000000 --- a/ociserver/registry.go +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package ociserver implements a docker V2 registry and the OCI distribution specification. -// -// It is designed to be used anywhere a low dependency container registry is needed. -// -// Its goal is to be standards compliant and its strictness will increase over time. -package ociserver - -import ( - "context" - "fmt" - "log" - "net/http" - "sync/atomic" - - "github.com/docker/oci" - "github.com/docker/oci/internal/ocirequest" -) - -// debug causes debug messages to be emitted when running the server. -const debug = false - -// Options holds options for the server. -type Options struct { - // WriteError is used to write error responses. It is passed the - // writer to write the error response to, the request that - // the error is in response to, and the error itself. - // - // If WriteError is nil, [oci.WriteError] will - // be used and any error discarded. - WriteError func(w http.ResponseWriter, req *http.Request, err error) - - // DisableReferrersAPI, when true, causes the registry to behave as if - // it does not understand the referrers API. - DisableReferrersAPI bool - - // DisableReferrersFiltering, when true, cause the registry - // to behave as if it does not recognize the artifactType filter - // on the referrers API. - DisableReferrersFiltering bool - - // DisableSinglePostUpload, when true, causes the registry - // to reject uploads with a single POST request. - // This is useful in combination with LocationsForDescriptor - // to cause uploaded blob content to flow through - // another server. - DisableSinglePostUpload bool - - // MaxListPageSize, if > 0, causes the list endpoints to return an - // error if the page size is greater than that. This emulates - // a quirk of AWS ECR where it refuses request for any - // page size > 1000. - MaxListPageSize int - - // OmitDigestFromTagGetResponse causes the registry - // to omit the Docker-Content-Digest header from a tag - // GET response, mimicking the behavior of registries that - // do the same (for example AWS ECR). - OmitDigestFromTagGetResponse bool - - // OmitLinkHeaderFromResponses causes the server - // to leave out the Link header from list responses. - OmitLinkHeaderFromResponses bool - - // LocationForUploadID transforms an upload ID as returned by - // ocirequest.BlobWriter.ID to the absolute URL location - // as returned by the upload endpoints. - // - // By default, when this function is nil, or it returns an empty - // string, upload IDs are treated as opaque identifiers and the - // returned locations are always host-relative URLs into the - // server itself. - // - // This can be used to allow clients to fetch and push content - // directly from some upstream server rather than passing - // through this server. Clients doing that will need access - // rights to that remote location. - LocationForUploadID func(string) (string, error) - - // LocationsForDescriptor returns a set of possible download - // URLs for the given descriptor. - // If it's nil, then all locations returned by the server - // will refer to the server itself. - // - // If not, then the Location header of responses will be - // set accordingly (to an arbitrary value from the - // returned slice if there are multiple). - // - // Returning a location from this function will also - // cause GET requests to return a redirect response - // to that location. - // - // TODO perhaps the redirect behavior described above - // isn't always what is wanted? - LocationsForDescriptor func(isManifest bool, desc oci.Descriptor) ([]string, error) - - DebugID string -} - -var debugID int32 - -// New returns a handler which implements the docker registry protocol -// by making calls to the underlying registry backend r. -// -// If opts is nil, it's equivalent to passing new(Options). -// -// The returned handler should be registered at the site root. -// -// # Errors -// -// All HTTP responses will be JSON, formatted according to the -// OCI spec. If an error returned from backend conforms to -// [oci.Error], the associated code and detail will be used. -// -// The HTTP response code will be determined from the error -// code when possible. If it can't be determined and the -// error implements [oci.HTTPError], the code returned -// by StatusCode will be used as the HTTP response code. -func New(backend oci.Interface, opts *Options) http.Handler { - if opts == nil { - opts = new(Options) - } - r := ®istry{ - opts: *opts, - backend: backend, - } - if r.opts.DebugID == "" { - r.opts.DebugID = fmt.Sprintf("ociserver%d", atomic.AddInt32(&debugID, 1)) - } - if r.opts.WriteError == nil { - r.opts.WriteError = func(w http.ResponseWriter, _ *http.Request, err error) { - oci.WriteError(w, err) - } - } - return r -} - -func (r *registry) logf(f string, a ...any) { - log.Printf("ociserver %s: %s", r.opts.DebugID, fmt.Sprintf(f, a...)) -} - -type registry struct { - opts Options - backend oci.Interface -} - -var handlers = []func(r *registry, ctx context.Context, w http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error{ - ocirequest.ReqPing: (*registry).handlePing, - ocirequest.ReqBlobGet: (*registry).handleBlobGet, - ocirequest.ReqBlobHead: (*registry).handleBlobHead, - ocirequest.ReqBlobDelete: (*registry).handleBlobDelete, - ocirequest.ReqBlobStartUpload: (*registry).handleBlobStartUpload, - ocirequest.ReqBlobUploadBlob: (*registry).handleBlobUploadBlob, - ocirequest.ReqBlobMount: (*registry).handleBlobMount, - ocirequest.ReqBlobUploadInfo: (*registry).handleBlobUploadInfo, - ocirequest.ReqBlobUploadChunk: (*registry).handleBlobUploadChunk, - ocirequest.ReqBlobCompleteUpload: (*registry).handleBlobCompleteUpload, - ocirequest.ReqManifestGet: (*registry).handleManifestGet, - ocirequest.ReqManifestHead: (*registry).handleManifestHead, - ocirequest.ReqManifestPut: (*registry).handleManifestPut, - ocirequest.ReqManifestDelete: (*registry).handleManifestDelete, - ocirequest.ReqTagsList: (*registry).handleTagsList, - ocirequest.ReqReferrersList: (*registry).handleReferrersList, - ocirequest.ReqCatalogList: (*registry).handleCatalogList, -} - -func (r *registry) ServeHTTP(resp http.ResponseWriter, req *http.Request) { - if rerr := r.v2(resp, req); rerr != nil { - r.opts.WriteError(resp, req, rerr) - return - } -} - -// https://docs.docker.com/registry/spec/api/#api-version-check -// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#api-version-check -func (r *registry) v2(resp http.ResponseWriter, req *http.Request) (_err error) { - if debug { - r.logf("registry.v2 %v %s {", req.Method, req.URL) - defer func() { - if _err != nil { - r.logf("} -> %v", _err) - } else { - r.logf("}") - } - }() - } - - rreq, err := ocirequest.Parse(req.Method, req.URL) - if err != nil { - resp.Header().Set("Docker-Distribution-API-Version", "registry/2.0") - return err - } - handle := handlers[rreq.Kind] - return handle(r, req.Context(), resp, req, rreq) -} - -func (r *registry) handlePing(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - resp.Header().Set("Docker-Distribution-API-Version", "registry/2.0") - return nil -} - -func (r *registry) setLocationHeader(resp http.ResponseWriter, isManifest bool, desc oci.Descriptor, defaultLocation string) error { - loc := defaultLocation - if r.opts.LocationsForDescriptor != nil { - locs, err := r.opts.LocationsForDescriptor(isManifest, desc) - if err != nil { - what := "blob" - if isManifest { - what = "manifest" - } - return fmt.Errorf("cannot determine location for %s: %v", what, err) - } - if len(locs) > 0 { - loc = locs[0] // TODO select arbitrary location from the slice - } - } - resp.Header().Set("Location", loc) - resp.Header().Set("Docker-Content-Digest", desc.Digest.String()) - return nil -} diff --git a/ociserver/registry_test.go b/ociserver/registry_test.go deleted file mode 100644 index 6cfe07a..0000000 --- a/ociserver/registry_test.go +++ /dev/null @@ -1,678 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ociserver_test - -import ( - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/docker/oci/ocidigest" - "github.com/docker/oci/ocimem" - "github.com/docker/oci/ociserver" - "github.com/stretchr/testify/require" -) - -const ( - weirdIndex = `{ - "manifests": [ - { - "size": 3, - "digest":"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - "mediaType":"application/vnd.oci.image.layer.nondistributable.v1.tar+gzip" - },{ - "size": 3, - "digest":"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - "mediaType":"application/xml" - },{ - "size": 3, - "digest":"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - "mediaType":"application/vnd.oci.image.manifest.v1+json" - } - ] -}` -) - -func TestCalls(t *testing.T) { - tcs := []struct { - skip bool - - Description string - - // Request / setup - Method string - Body string // request body to send - URL string - Digests map[string]string - Manifests map[string]string - BlobStream map[string]string - RequestHeader map[string]string - - // Response - WantCode int - WantHeader map[string]string - WantBody string // response body to expect - }{ - { - Description: "v2_returns_200", - Method: "GET", - URL: "/v2", - WantCode: http.StatusOK, - WantHeader: map[string]string{"Docker-Distribution-API-Version": "registry/2.0"}, - }, - { - Description: "v2_slash_returns_200", - Method: "GET", - URL: "/v2/", - WantCode: http.StatusOK, - WantHeader: map[string]string{"Docker-Distribution-API-Version": "registry/2.0"}, - }, - { - Description: "v2_bad_returns_404", - Method: "GET", - URL: "/v2/bad", - WantCode: http.StatusNotFound, - WantHeader: map[string]string{"Docker-Distribution-API-Version": "registry/2.0"}, - WantBody: `{"errors":[{"code":"UNKNOWN","message":"page not found"}]}`, - }, - { - Description: "GET_non_existent_blob", - Method: "GET", - URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - WantCode: http.StatusNotFound, - WantBody: `{"errors":[{"code":"NAME_UNKNOWN","message":"repository name not known to registry"}]}`, - }, - { - Description: "HEAD_non_existent_blob", - Method: "HEAD", - URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - WantCode: http.StatusNotFound, - }, - { - Description: "GET_bad_digest", - Method: "GET", - URL: "/v2/foo/blobs/sha256:asd", - WantCode: http.StatusBadRequest, - WantBody: `{"errors":[{"code":"DIGEST_INVALID","message":"badly formed digest"}]}`, - }, - { - Description: "HEAD_bad_digest", - Method: "HEAD", - URL: "/v2/foo/blobs/sha256:asd", - WantCode: http.StatusBadRequest, - }, - { - Description: "bad_blob_verb", - Method: "FOO", - URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - WantCode: http.StatusMethodNotAllowed, - WantBody: `{"errors":[{"code":"UNKNOWN","message":"method not allowed"}]}`, - }, - { - Description: "GET_containerless_blob", - Digests: map[string]string{"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae": "foo"}, - Method: "GET", - URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - WantCode: http.StatusOK, - WantHeader: map[string]string{"Docker-Content-Digest": "sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"}, - WantBody: "foo", - }, - { - Description: "GET_blob", - Digests: map[string]string{"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae": "foo"}, - Method: "GET", - URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - WantCode: http.StatusOK, - WantHeader: map[string]string{"Docker-Content-Digest": "sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"}, - WantBody: "foo", - }, - { - Description: "GET_blob_range_defined_range", - Digests: map[string]string{ - "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9": "hello world", - }, - Method: "GET", - RequestHeader: map[string]string{ - "Range": "bytes=1-4", - }, - URL: "/v2/foo/blobs/sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", - WantCode: http.StatusPartialContent, - WantHeader: map[string]string{ - "Docker-Content-Digest": "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", - "Content-Length": "4", - "Content-Range": "bytes 1-4/11", - }, - WantBody: "ello", - }, - { - Description: "GET_blob_range_undefined_range_end", - Digests: map[string]string{ - "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9": "hello world", - }, - Method: "GET", - RequestHeader: map[string]string{ - "Range": "bytes=3-", - }, - URL: "/v2/foo/blobs/sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", - WantCode: http.StatusPartialContent, - WantHeader: map[string]string{ - "Docker-Content-Digest": "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", - "Content-Length": "8", - "Content-Range": "bytes 3-10/11", - }, - WantBody: "lo world", - }, - { - Description: "GET_blob_range_invalid-range-start", - Digests: map[string]string{ - "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9": "hello world", - }, - Method: "GET", - RequestHeader: map[string]string{ - "Range": "bytes=20-30", - }, - URL: "/v2/foo/blobs/sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", - // TODO change ocimem to return an error that results in a 416 status. - WantCode: http.StatusInternalServerError, - WantBody: `{"errors":[{"code":"UNKNOWN","message":"invalid range [20, 11]; have [0, 11]"}]}`, - }, - { - Description: "HEAD_blob", - Digests: map[string]string{"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae": "foo"}, - Method: "HEAD", - URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - WantCode: http.StatusOK, - WantHeader: map[string]string{ - "Content-Length": "3", - "Docker-Content-Digest": "sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - }, - }, - { - Description: "DELETE_blob", - Digests: map[string]string{"sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae": "foo"}, - Method: "DELETE", - URL: "/v2/foo/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - WantCode: http.StatusAccepted, - }, - { - Description: "blob_url_with_no_container", - Method: "GET", - URL: "/v2/blobs/sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - WantCode: http.StatusNotFound, - WantBody: `{"errors":[{"code":"UNKNOWN","message":"page not found"}]}`, - }, - { - Description: "uploadurl", - Method: "POST", - URL: "/v2/foo/blobs/uploads/", - WantCode: http.StatusAccepted, - WantHeader: map[string]string{"Range": "0-0"}, - }, - { - Description: "uploadurl", - Method: "POST", - URL: "/v2/foo/blobs/uploads/", - WantCode: http.StatusAccepted, - WantHeader: map[string]string{"Range": "0-0"}, - }, - { - Description: "upload_put_missing_digest", - Method: "PUT", - URL: "/v2/foo/blobs/uploads/MQ", - WantCode: http.StatusBadRequest, - WantBody: `{"errors":[{"code":"DIGEST_INVALID","message":"badly formed digest"}]}`, - }, - { - Description: "monolithic_upload_good_digest", - Method: "POST", - URL: "/v2/foo/blobs/uploads?digest=sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - WantCode: http.StatusCreated, - Body: "foo", - WantHeader: map[string]string{"Docker-Content-Digest": "sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"}, - }, - { - Description: "monolithic_upload_bad_digest", - Method: "POST", - URL: "/v2/foo/blobs/uploads?digest=sha256:fake", - Body: "foo", - WantCode: http.StatusBadRequest, - WantBody: `{"errors":[{"code":"DIGEST_INVALID","message":"badly formed digest"}]}`, - }, - { - Description: "upload_good_digest", - Method: "PUT", - URL: "/v2/foo/blobs/uploads/MQ?digest=sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - WantCode: http.StatusCreated, - Body: "foo", - WantHeader: map[string]string{"Docker-Content-Digest": "sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"}, - }, - { - Description: "upload_bad_digest", - Method: "PUT", - URL: "/v2/foo/blobs/uploads/MQ?digest=sha256:baddigest", - WantCode: http.StatusBadRequest, - Body: "foo", - WantBody: `{"errors":[{"code":"DIGEST_INVALID","message":"badly formed digest"}]}`, - }, - { - Description: "stream_upload", - Method: "PATCH", - URL: "/v2/foo/blobs/uploads/MQ", - WantCode: http.StatusAccepted, - Body: "foo", - RequestHeader: map[string]string{ - "Content-Range": "0-2", - }, - WantHeader: map[string]string{ - "Range": "0-2", - "Location": "/v2/foo/blobs/uploads/MQ", - }, - }, - { - skip: true, - Description: "stream_duplicate_upload", - Method: "PATCH", - URL: "/v2/foo/blobs/uploads/MQ", - WantCode: http.StatusBadRequest, - Body: "foo", - BlobStream: map[string]string{"MQ": "foo"}, - }, - { - Description: "stream_finish_upload", - Method: "PUT", - URL: "/v2/foo/blobs/uploads/MQ?digest=sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", - BlobStream: map[string]string{"MQ": "foo"}, - WantCode: http.StatusCreated, - WantHeader: map[string]string{"Docker-Content-Digest": "sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"}, - }, - { - Description: "get_missing_manifest", - Method: "GET", - URL: "/v2/foo/manifests/latest", - WantCode: http.StatusNotFound, - WantBody: `{"errors":[{"code":"NAME_UNKNOWN","message":"repository name not known to registry"}]}`, - }, - { - Description: "head_missing_manifest", - Method: "HEAD", - URL: "/v2/foo/manifests/latest", - WantCode: http.StatusNotFound, - }, - { - Description: "get_missing_manifest_good_container", - Manifests: map[string]string{"foo/manifests/latest": "foo"}, - Method: "GET", - URL: "/v2/foo/manifests/bar", - WantCode: http.StatusNotFound, - WantBody: `{"errors":[{"code":"MANIFEST_UNKNOWN","message":"manifest unknown to registry"}]}`, - }, - { - Description: "head_missing_manifest_good_container", - Manifests: map[string]string{"foo/manifests/latest": "foo"}, - Method: "HEAD", - URL: "/v2/foo/manifests/bar", - WantCode: http.StatusNotFound, - }, - { - Description: "get_manifest_by_tag", - Manifests: map[string]string{"foo/manifests/latest": "foo"}, - Method: "GET", - URL: "/v2/foo/manifests/latest", - WantCode: http.StatusOK, - WantBody: "foo", - }, - { - Description: "get_manifest_by_digest", - Manifests: map[string]string{"foo/manifests/latest": "foo"}, - Method: "GET", - URL: "/v2/foo/manifests/" + digestOf("foo"), - WantCode: http.StatusOK, - WantBody: "foo", - }, - { - Description: "head_manifest", - Manifests: map[string]string{"foo/manifests/latest": "foo"}, - Method: "HEAD", - URL: "/v2/foo/manifests/latest", - WantCode: http.StatusOK, - }, - { - Description: "create_manifest", - Method: "PUT", - URL: "/v2/foo/manifests/latest", - WantCode: http.StatusCreated, - Body: "foo", - }, - { - Description: "create_index", - Method: "PUT", - URL: "/v2/foo/manifests/latest", - WantCode: http.StatusCreated, - Body: weirdIndex, - RequestHeader: map[string]string{ - "Content-Type": "application/vnd.oci.image.index.v1+json", - }, - Manifests: map[string]string{"foo/manifests/image": "foo"}, - }, - { - skip: true, - Description: "create_index_missing_child", - Method: "PUT", - URL: "/v2/foo/manifests/latest", - WantCode: http.StatusNotFound, - Body: weirdIndex, - RequestHeader: map[string]string{ - "Content-Type": "application/vnd.oci.image.index.v1+json", - }, - }, - { - skip: true, - Description: "bad_index_body", - Method: "PUT", - URL: "/v2/foo/manifests/latest", - WantCode: http.StatusBadRequest, - Body: "foo", - RequestHeader: map[string]string{ - "Content-Type": "application/vnd.oci.image.index.v1+json", - }, - }, - { - Description: "bad_manifest_method", - Method: "BAR", - URL: "/v2/foo/manifests/latest", - WantCode: http.StatusMethodNotAllowed, - WantBody: `{"errors":[{"code":"UNKNOWN","message":"method not allowed"}]}`, - }, - { - Description: "Chunk_upload_start", - Method: "PATCH", - URL: "/v2/foo/blobs/uploads/MQ", - RequestHeader: map[string]string{"Content-Range": "0-2"}, - WantCode: http.StatusAccepted, - Body: "foo", - WantHeader: map[string]string{ - "Range": "0-2", - "Location": "/v2/foo/blobs/uploads/MQ", - }, - }, - { - Description: "Chunk_upload_bad_content_range", - Method: "PATCH", - URL: "/v2/foo/blobs/uploads/MQ", - RequestHeader: map[string]string{"Content-Range": "0-bar"}, - // TODO the original had 405 response here. Which is correct? - WantCode: http.StatusBadRequest, - Body: "foo", - WantBody: `{"errors":[{"code":"UNSUPPORTED","message":"we don't understand your Content-Range"}]}`, - }, - { - Description: "Chunk_upload_overlaps_previous_data", - Method: "PATCH", - URL: "/v2/foo/blobs/uploads/MQ", - BlobStream: map[string]string{"MQ": "foo"}, - RequestHeader: map[string]string{"Content-Range": "2-4"}, - WantCode: http.StatusRequestedRangeNotSatisfiable, - Body: "bar", - WantBody: `{"errors":[{"code":"RANGE_INVALID","message":"cannot copy blob data: invalid offset 2 in resumed upload (actual offset 3): range invalid: invalid content range"}]}`, - }, - { - Description: "Chunk_upload_after_previous_data", - Method: "PATCH", - URL: "/v2/foo/blobs/uploads/MQ", - BlobStream: map[string]string{"MQ": "foo"}, - RequestHeader: map[string]string{"Content-Range": "3-5"}, - WantCode: http.StatusAccepted, - Body: "bar", - WantHeader: map[string]string{ - "Range": "0-5", - "Location": "/v2/foo/blobs/uploads/MQ", - }, - }, - { - Description: "DELETE_Unknown_name", - Method: "DELETE", - URL: "/v2/test/honk/manifests/latest", - WantCode: http.StatusNotFound, - WantBody: `{"errors":[{"code":"NAME_UNKNOWN","message":"repository name not known to registry"}]}`, - }, - { - Description: "DELETE_Unknown_manifest", - Manifests: map[string]string{"honk/manifests/latest": "honk"}, - Method: "DELETE", - URL: "/v2/honk/manifests/tag-honk", - WantCode: http.StatusNotFound, - WantBody: `{"errors":[{"code":"MANIFEST_UNKNOWN","message":"manifest unknown to registry: tag does not exist"}]}`, - }, - { - Description: "DELETE_existing_manifest", - Manifests: map[string]string{"foo/manifests/latest": "foo"}, - Method: "DELETE", - URL: "/v2/foo/manifests/latest", - WantCode: http.StatusAccepted, - }, - { - Description: "DELETE_existing_manifest_by_digest", - Manifests: map[string]string{"foo/manifests/latest": "foo"}, - Method: "DELETE", - URL: "/v2/foo/manifests/" + digestOf("foo"), - WantCode: http.StatusAccepted, - }, - { - Description: "list_tags", - Manifests: map[string]string{"foo/manifests/latest": "foo", "foo/manifests/tag1": "foo"}, - Method: "GET", - URL: "/v2/foo/tags/list?n=1000", - WantCode: http.StatusOK, - WantBody: `{"name":"foo","tags":["latest","tag1"]}`, - }, - { - Description: "limit_tags", - Manifests: map[string]string{"foo/manifests/latest": "foo", "foo/manifests/tag1": "foo"}, - Method: "GET", - URL: "/v2/foo/tags/list?n=1", - WantCode: http.StatusOK, - WantBody: `{"name":"foo","tags":["latest"]}`, - }, - { - Description: "offset_tags", - Manifests: map[string]string{"foo/manifests/latest": "foo", "foo/manifests/tag1": "foo"}, - Method: "GET", - URL: "/v2/foo/tags/list?last=latest", - WantCode: http.StatusOK, - WantBody: `{"name":"foo","tags":["tag1"]}`, - }, - { - Description: "list_non_existing_tags", - Method: "GET", - URL: "/v2/foo/tags/list?n=1000", - WantCode: http.StatusNotFound, - WantBody: `{"errors":[{"code":"NAME_UNKNOWN","message":"repository name not known to registry"}]}`, - }, - { - Description: "list_tags_bad_request", - Method: "GET", - URL: "/v2/foo/tags/list?n=INVALID", - WantCode: http.StatusBadRequest, - WantBody: `{"errors":[{"code":"UNKNOWN","message":"query parameter n is not a valid integer"}]}`, - }, - { - Description: "list_repos", - Manifests: map[string]string{"foo/manifests/latest": "foo", "bar/manifests/latest": "bar"}, - Method: "GET", - URL: "/v2/_catalog?n=1000", - WantCode: http.StatusOK, - WantBody: `{"repositories":["bar","foo"]}`, - }, - { - Description: "fetch_references", - Method: "GET", - URL: "/v2/foo/referrers/" + digestOf("foo"), - WantCode: http.StatusOK, - Manifests: map[string]string{ - "foo/manifests/image": "foo", - "foo/manifests/points-to-image": "{\"subject\": {\"digest\": \"" + digestOf("foo") + "\"}}", - }, - WantBody: `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":null}`, - }, - { - Description: "fetch_references,_subject_pointing_elsewhere", - Method: "GET", - URL: "/v2/foo/referrers/" + digestOf("foo"), - WantCode: http.StatusOK, - Manifests: map[string]string{ - "foo/manifests/image": "foo", - "foo/manifests/points-to-image": "{\"subject\": {\"digest\": \"" + digestOf("nonexistant") + "\"}}", - }, - WantBody: `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":null}`, - }, - { - Description: "fetch_references,_no_results", - Method: "GET", - URL: "/v2/foo/referrers/" + digestOf("foo"), - WantCode: http.StatusOK, - Manifests: map[string]string{ - "foo/manifests/image": "foo", - }, - WantBody: `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":null}`, - }, - { - Description: "fetch_references,_missing_repo", - Method: "GET", - URL: "/v2/does-not-exist/referrers/" + digestOf("foo"), - WantCode: http.StatusNotFound, - WantBody: `{"errors":[{"code":"NAME_UNKNOWN","message":"repository name not known to registry"}]}`, - }, - { - Description: "fetch_references,_bad_target_(tag_vs._digest)", - Method: "GET", - URL: "/v2/foo/referrers/latest", - WantCode: http.StatusBadRequest, - WantBody: `{"errors":[{"code":"DIGEST_INVALID","message":"badly formed digest"}]}`, - }, - { - skip: true, - Description: "fetch_references,_bad_method", - Method: "POST", - URL: "/v2/foo/referrers/" + digestOf("foo"), - WantCode: http.StatusBadRequest, - }, - } - - for _, tc := range tcs { - - testf := func(t *testing.T) { - if tc.skip { - t.Skip("skipping") - } - r := ociserver.New(ocimem.New(), nil) - s := httptest.NewServer(r) - defer s.Close() - - for manifest, contents := range tc.Manifests { - req, _ := http.NewRequest("PUT", s.URL+"/v2/"+manifest, strings.NewReader(contents)) - req.Header.Set("Content-Type", "application/octet-stream") // TODO better media type - t.Log(req.Method, req.URL) - resp, err := s.Client().Do(req) - if err != nil { - t.Fatalf("Error uploading manifest: %v", err) - } - if resp.StatusCode != http.StatusCreated { - body, _ := io.ReadAll(resp.Body) - t.Fatalf("Error uploading manifest got status: %d %s", resp.StatusCode, body) - } - t.Logf("created manifest with digest %v", resp.Header.Get("Docker-Content-Digest")) - } - - for digest, contents := range tc.Digests { - req, _ := http.NewRequest( - "POST", - fmt.Sprintf("%s/v2/foo/blobs/uploads/?digest=%s", s.URL, digest), - strings.NewReader(contents), - ) - req.Header.Set("Content-Length", fmt.Sprint(len(contents))) // TODO better media type - t.Log(req.Method, req.URL) - resp, err := s.Client().Do(req) - if err != nil { - t.Fatalf("Error uploading digest: %v", err) - } - if resp.StatusCode != http.StatusCreated { - body, _ := io.ReadAll(resp.Body) - t.Fatalf("Error uploading digest got status: %d %s", resp.StatusCode, body) - } - } - - for upload, contents := range tc.BlobStream { - req, err := http.NewRequest( - "PATCH", - fmt.Sprintf("%s/v2/foo/blobs/uploads/%s", s.URL, upload), - io.NopCloser(strings.NewReader(contents)), - ) - if err != nil { - t.Fatal(err) - } - req.Header.Add("Content-Range", fmt.Sprintf("0-%d", len(contents)-1)) - t.Log(req.Method, req.URL) - resp, err := s.Client().Do(req) - if err != nil { - t.Fatalf("Error streaming blob: %v", err) - } - if resp.StatusCode != http.StatusAccepted { - body, _ := io.ReadAll(resp.Body) - t.Fatalf("Error streaming blob: %d %s", resp.StatusCode, body) - } - - } - - req, err := http.NewRequest(tc.Method, s.URL+tc.URL, strings.NewReader(tc.Body)) - require.NoError(t, err) - for k, v := range tc.RequestHeader { - req.Header.Set(k, v) - } - t.Logf("%s %v", req.Method, req.URL) - resp, err := s.Client().Do(req) - if err != nil { - t.Fatalf("Error getting %q: %v", tc.URL, err) - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - t.Errorf("Reading response body: %v", err) - } - if resp.StatusCode != tc.WantCode { - t.Fatalf("Incorrect status code, got %d, want %d; body: %s", resp.StatusCode, tc.WantCode, body) - } - - for k, v := range tc.WantHeader { - r := resp.Header.Get(k) - if r != v { - t.Errorf("Incorrect header %q received, got %q, want %q", k, r, v) - } - } - - if string(body) != tc.WantBody { - t.Logf("\n WantBody: `%s`,", body) - t.Errorf("Incorrect response body.\ngot:\n\t%q\n\twant:\n\t%q", body, tc.WantBody) - } - } - t.Run(tc.Description, testf) - } -} - -func digestOf(s string) string { - return ocidigest.FromBytes([]byte(s)).String() -} diff --git a/ociserver/router_test.go b/ociserver/router_test.go new file mode 100644 index 0000000..baad674 --- /dev/null +++ b/ociserver/router_test.go @@ -0,0 +1,16 @@ +package ociserver + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/docker/oci/ociserver/mux" +) + +func serveTestRoute(t testing.TB, pattern string, handler http.Handler, rec *httptest.ResponseRecorder, req *http.Request) { + t.Helper() + r := mux.New() + r.Handle(pattern, handler) + r.ServeHTTP(rec, req) +} diff --git a/ociserver/server.go b/ociserver/server.go new file mode 100644 index 0000000..91695f9 --- /dev/null +++ b/ociserver/server.go @@ -0,0 +1,175 @@ +package ociserver + +import ( + "context" + "encoding/json" + "log/slog" + "net/http" + "slices" + "strings" + + "github.com/docker/oci" + "github.com/docker/oci/ociref" + "github.com/docker/oci/ociserver/mux" +) + +// ServerConfig configures a [Server]. +type ServerConfig struct { + // Logger receives diagnostics for unexpected internal failures. A nil + // Logger disables logging. + Logger *slog.Logger + + // Middlewares are applied to all routes on the server's root mux, including + // routes added through [Server.Mux]. They are applied in the order provided; + // the first middleware is the outermost middleware. + Middlewares []func(http.Handler) http.Handler + + // AuthMiddleware, when non-nil, is applied only to the built-in OCI routes. + AuthMiddleware func(http.Handler) http.Handler +} + +// Server contains everything necessary to run the API portion of the service +type Server struct { + cfg ServerConfig + mux *mux.Mux + db oci.Interface + redirect Redirecter +} + +var _ http.Handler = (*Server)(nil) + +// New returns a new server backed by pers. A nil cfg is equivalent to a +// pointer to a zero [ServerConfig]. The configuration is copied before New +// returns and may be modified by the caller afterward. +func New(pers oci.Interface, cfg0 *ServerConfig) (*Server, error) { + var cfg ServerConfig + if cfg0 != nil { + cfg = *cfg0 + cfg.Middlewares = slices.Clone(cfg0.Middlewares) + } + s := &Server{ + cfg: cfg, + db: pers, + } + if redirecter, ok := pers.(Redirecter); ok { + s.redirect = redirecter + } + s.addRoutes() + return s, nil +} + +// ServeHTTP implements [http.Handler]. +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mux.ServeHTTP(w, r) +} + +// Mux returns the server's root router. Callers may use it to add routes outside +// the built-in /v2 namespace, which is owned by a mounted child router. Root +// middleware must be configured through [ServerConfig.Middlewares] because the +// router does not allow middleware to be added after routes are registered. +func (s *Server) Mux() *mux.Mux { + return s.mux +} + +func (s *Server) logError(ctx context.Context, message string, err error, args ...any) { + if s.cfg.Logger == nil { + return + } + safeArgs := make([]any, len(args)) + copy(safeArgs, args) + for i, arg := range safeArgs { + if str, ok := arg.(string); ok { + safeArgs[i] = sanitizeLogInput(str) + } + } + s.cfg.Logger.ErrorContext(ctx, message, append(safeArgs, "error", err)...) +} + +// sanitizeLogInput replaces newline characters to prevent log injection +func sanitizeLogInput(input string) string { + escaped := strings.NewReplacer("\n", "\\n", "\r", "\\r") + return escaped.Replace(input) +} + +func (s *Server) addRoutes() { + r := mux.New() + if len(s.cfg.Middlewares) > 0 { + r.Use(s.cfg.Middlewares...) + } + + r.Route("/v2", func(r mux.Router) { + if s.cfg.AuthMiddleware != nil { + r.Use(s.cfg.AuthMiddleware) + } + r.Use(validateRepositoryName) + + r.Handle("/", s.root()) + + // blob uploads + r.Post("/*name/blobs/uploads/", s.blobUploadPost()) + r.Get("/*name/blobs/uploads/:session", s.blobUploadGet()) + r.Patch("/*name/blobs/uploads/:session", s.blobUploadPatch()) + r.Put("/*name/blobs/uploads/:session", s.blobUploadPut()) + r.Delete("/*name/blobs/uploads/:session", s.blobUploadDelete()) + + // blobs + r.Head("/*name/blobs/:digest", s.blobHeadGet()) + r.Get("/*name/blobs/:digest", s.blobHeadGet()) + r.Delete("/*name/blobs/:digest", s.blobDelete()) + + // manifests + r.Head("/*name/manifests/:reference", s.manifestHeadGet()) + r.Get("/*name/manifests/:reference", s.manifestHeadGet()) + r.Put("/*name/manifests/:reference", s.manifestPut()) + r.Delete("/*name/manifests/:reference", s.manifestDelete()) + + // tags + r.Get("/*name/tags/list", s.tagsGet()) + + // referrers + r.Get("/*name/referrers/:digest", s.referrersGet()) + }) + s.mux = r +} + +func validateRepositoryName(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + name := mux.URLParam(r, "name") + if name != "" && !ociref.IsValidRepository(name) { + returnError(w, ErrNameInvalid(name)) + return + } + next.ServeHTTP(w, r) + }) +} + +func (s *Server) root() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + } +} + +func returnError(w http.ResponseWriter, oe *OCIError) { + b, err := json.Marshal(map[string]any{ + "errors": []*OCIError{oe}, + }) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + if oe.status == http.StatusRequestedRangeNotSatisfiable { + w.Header().Set("Content-Range", "bytes */*") + } + w.WriteHeader(oe.status) + _, _ = w.Write(b) +} + +// Redirecter provides storage-backed URL generation for blob redirects. +type Redirecter interface { + // Redirect returns whether the request should redirect, the redirect URL, and + // any error encountered while deciding. + Redirect(r *http.Request, desc oci.Descriptor, repo string) (bool, string, error) +} diff --git a/ociserver/server_test.go b/ociserver/server_test.go new file mode 100644 index 0000000..e7fd409 --- /dev/null +++ b/ociserver/server_test.go @@ -0,0 +1,148 @@ +package ociserver + +import ( + "bytes" + "context" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/docker/oci" + "github.com/docker/oci/ocidigest" + "github.com/stretchr/testify/require" +) + +func TestServerDefaultLoggingIsSilent(t *testing.T) { + var globalLog bytes.Buffer + oldDefault := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&globalLog, nil))) + t.Cleanup(func() { + slog.SetDefault(oldDefault) + }) + + srv, err := New(&oci.Funcs{ + ResolveBlob_: func(context.Context, string, oci.Digest) (oci.Descriptor, error) { + return oci.Descriptor{}, errors.New("backend failed") + }, + }, nil) + require.NoError(t, err) + + dgst := ocidigest.FromBytes([]byte("blob")) + req := httptest.NewRequest(http.MethodHead, "/v2/example/blobs/"+dgst.String(), nil) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + require.Equal(t, http.StatusInternalServerError, rec.Code) + require.Empty(t, globalLog.String()) +} + +func TestServerLogsUnexpectedFailures(t *testing.T) { + var logs bytes.Buffer + logger := slog.New(slog.NewTextHandler(&logs, nil)) + srv, err := New(&oci.Funcs{ + ResolveBlob_: func(context.Context, string, oci.Digest) (oci.Descriptor, error) { + return oci.Descriptor{}, errors.New("backend failed") + }, + }, &ServerConfig{Logger: logger}) + require.NoError(t, err) + + dgst := ocidigest.FromBytes([]byte("blob")) + req := httptest.NewRequest(http.MethodHead, "/v2/example/blobs/"+dgst.String(), nil) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + require.Equal(t, http.StatusInternalServerError, rec.Code) + require.Contains(t, logs.String(), "msg=\"resolving blob\"") + require.Contains(t, logs.String(), "repository=example") + require.Contains(t, logs.String(), "error=\"backend failed\"") +} + +func TestServerMiddlewares(t *testing.T) { + var calls []string + middleware := func(name string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, name+" before") + next.ServeHTTP(w, r) + calls = append(calls, name+" after") + }) + } + } + cfg := &ServerConfig{ + Middlewares: []func(http.Handler) http.Handler{ + middleware("first"), + middleware("second"), + }, + } + srv, err := New((*oci.Funcs)(nil), cfg) + require.NoError(t, err) + + // New must retain its own middleware slice rather than the caller's backing + // array. + cfg.Middlewares[0] = func(http.Handler) http.Handler { + panic("server used the caller's mutated middleware slice") + } + + req := httptest.NewRequest(http.MethodGet, "/v2/", nil) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, []string{ + "first before", + "second before", + "second after", + "first after", + }, calls) + require.True(t, strings.HasPrefix(rec.Header().Get("Content-Type"), "application/json")) +} + +func TestServerAuthMiddlewareOnlyAppliesToOCIRoutes(t *testing.T) { + var calls []string + middleware := func(name string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, name) + next.ServeHTTP(w, r) + }) + } + } + srv, err := New((*oci.Funcs)(nil), &ServerConfig{ + Middlewares: []func(http.Handler) http.Handler{middleware("root")}, + AuthMiddleware: middleware("auth"), + }) + require.NoError(t, err) + srv.Mux().Get(`/health`, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v2/", nil)) + require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, []string{"root", "auth"}, calls) + + calls = nil + rec = httptest.NewRecorder() + srv.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/health", nil)) + require.Equal(t, http.StatusNoContent, rec.Code) + require.Equal(t, []string{"root"}, calls) +} + +func TestServerInvalidRepositoryNameReturnsOCIError(t *testing.T) { + srv, err := New((*oci.Funcs)(nil), nil) + require.NoError(t, err) + + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v2/INVALID/manifests/latest", nil)) + + require.Equal(t, http.StatusBadRequest, rec.Code) + require.JSONEq(t, `{ + "errors": [{ + "code": "NAME_INVALID", + "message": "repository name is invalid: INVALID" + }] + }`, rec.Body.String()) +} diff --git a/ociserver/tags.go b/ociserver/tags.go new file mode 100644 index 0000000..efbbdc9 --- /dev/null +++ b/ociserver/tags.go @@ -0,0 +1,62 @@ +package ociserver + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + + "github.com/docker/oci" + "github.com/docker/oci/ociserver/mux" +) + +func (s *Server) tagsGet() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := mux.URLParam(r, "name") + last := r.URL.Query().Get("last") + limit := 0 + if n := r.URL.Query().Get("n"); n != "" { + i, err := strconv.Atoi(n) + if err != nil || i < 0 { + returnError(w, ErrBadRequest("invalid n")) + return + } + limit = i + } + + // Fetch one extra tag to determine whether a next page exists without + // relying on len(tags)==limit, which can give a false positive when the + // total tag count is an exact multiple of the page size. + fetchLimit := limit + if limit > 0 { + fetchLimit = limit + 1 + } + tagSeq := s.db.Tags(r.Context(), name, &oci.TagsParameters{StartAfter: last, Limit: fetchLimit}) + tags, err := oci.All[string](tagSeq) + if err != nil { + if errors.Is(err, oci.ErrNameUnknown) { + returnError(w, ErrNameUnknown(name)) + return + } + s.logError(r.Context(), "listing tags", err, "repository", name) + returnError(w, ErrServerError()) + return + } + if limit > 0 && len(tags) > limit { + tags = tags[:limit] + lastTag := tags[len(tags)-1] + link := fmt.Sprintf(`; rel="next"`, name, url.QueryEscape(lastTag), limit) + w.Header().Set("Link", link) + } + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(map[string]any{ + "name": name, + "tags": tags, + }) + if err != nil { + s.logError(r.Context(), "writing tags response", err, "repository", name) + } + } +} diff --git a/ociserver/writer.go b/ociserver/writer.go deleted file mode 100644 index 99c45a3..0000000 --- a/ociserver/writer.go +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ociserver - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strconv" - - "github.com/docker/oci" - - "github.com/docker/oci/internal/ocirequest" - "github.com/docker/oci/ocidigest" -) - -func (r *registry) handleBlobUploadBlob(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - if r.opts.DisableSinglePostUpload { - return r.handleBlobStartUpload(ctx, resp, req, rreq) - } - // TODO check that Content-Type is application/octet-stream? - mediaType := mediaTypeOctetStream - - digest, err := ocidigest.Parse(rreq.Digest) - if err != nil { - return err - } - if digest == "" { - return oci.NewError("badly formed digest", oci.ErrDigestInvalid.Code(), nil) - } - desc, err := r.backend.PushBlob(req.Context(), rreq.Repo, oci.Descriptor{ - MediaType: mediaType, - Size: req.ContentLength, - Digest: digest, - }, req.Body) - if err != nil { - return err - } - if err := r.setLocationHeader(resp, false, desc, "/v2/"+rreq.Repo+"/blobs/"+desc.Digest.String()); err != nil { - return err - } - resp.WriteHeader(http.StatusCreated) - return nil -} - -func (r *registry) handleBlobStartUpload(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - // Start a chunked upload. When r.backend is ociclient, this should - // just result in a single POST request that starts the upload. - w, err := r.backend.PushBlobChunked(ctx, rreq.Repo, 0) - if err != nil { - return err - } - defer w.Close() - - resp.Header().Set("Location", r.locationForUploadID(rreq.Repo, w.ID())) - resp.Header().Set("Range", "0-0") - // TODO: reject chunks which don't follow this minimum length. - // If any reasonable clients are broken by this, we can always reconsider, - // perhaps by making the strictness on chunk sizes opt-in. - resp.Header().Set("OCI-Chunk-Min-Length", strconv.Itoa(w.ChunkSize())) - resp.WriteHeader(http.StatusAccepted) - return nil -} - -func (r *registry) handleBlobUploadInfo(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - // Resume the upload without actually writing to it, passing -1 for the offset - // to cause the backend to retrieve the associated upload information. - // When r.backend is ociclient, this should result in a single GET request - // to retrieve upload info. - w, err := r.backend.PushBlobChunkedResume(ctx, rreq.Repo, rreq.UploadID, -1, 0) - if err != nil { - return err - } - defer w.Close() - resp.Header().Set("Location", r.locationForUploadID(rreq.Repo, w.ID())) - resp.Header().Set("Range", ocirequest.RangeString(0, w.Size())) - resp.WriteHeader(http.StatusNoContent) - return nil -} - -func (r *registry) handleBlobUploadChunk(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - // Note that the spec requires chunked upload PATCH requests to include Content-Range, - // but the conformance tests do not actually follow that as of the time of writing. - // Allow the missing header to result in start=0, meaning we assume it's the first chunk. - start, end, err := chunkRange(req) - if err != nil { - return err - } - - w, err := r.backend.PushBlobChunkedResume(ctx, rreq.Repo, rreq.UploadID, start, int(end-start)) - if err != nil { - return err - } - if _, err := io.Copy(w, req.Body); err != nil { - w.Close() - return fmt.Errorf("cannot copy blob data: %w", err) - } - if err := w.Close(); err != nil { - return fmt.Errorf("cannot close BlobWriter: %w", err) - } - resp.Header().Set("Location", r.locationForUploadID(rreq.Repo, w.ID())) - resp.Header().Set("Range", ocirequest.RangeString(0, w.Size())) - resp.WriteHeader(http.StatusAccepted) - return nil -} - -func (r *registry) handleBlobCompleteUpload(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - // We are handling a PUT as part of one of: - // - // 1) An entire blob via POST-then-PUT. - // 2) The last chunk of a chunked upload as part of the closing PUT, with a valid Content-Range. - // 3) Closing a finished chunked upload with an empty-bodied PUT. - // - // We can't actually tell these apart upfront; - // for example, 3 can have an octet-stream content type even though it has no body, - // meaning that it looks exactly like 1, as seen in the conformance tests. - // For that reason, we simply forward the range start as the offset in case 2, - // while using an offset of 0 in cases 1 and 3 without a range, to avoid a GET in ociclient. - // - // Note that we don't check "ok" here, letting "start" default to 0 due to the above. - start, end, err := chunkRange(req) - if err != nil { - return err - } - - w, err := r.backend.PushBlobChunkedResume(ctx, rreq.Repo, rreq.UploadID, start, int(end-start)) - if err != nil { - return err - } - defer w.Close() - - if _, err := io.Copy(w, req.Body); err != nil { - return fmt.Errorf("failed to copy data to %T: %v", w, err) - } - digest, err := ocidigest.Parse(rreq.Digest) - if err != nil { - return err - } - if digest == "" { - return oci.NewError("badly formed digest", oci.ErrDigestInvalid.Code(), nil) - } - desc, err := w.Commit(digest) - if err != nil { - return err - } - if err := r.setLocationHeader(resp, false, desc, "/v2/"+rreq.Repo+"/blobs/"+desc.Digest.String()); err != nil { - return err - } - resp.WriteHeader(http.StatusCreated) - return nil -} - -func (r *registry) handleBlobMount(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - digest, err := ocidigest.Parse(rreq.Digest) - if err != nil { - return err - } - if digest == "" { - return oci.NewError("badly formed digest", oci.ErrDigestInvalid.Code(), nil) - } - desc, err := r.backend.MountBlob(ctx, rreq.FromRepo, rreq.Repo, digest) - if err != nil { - return err - } - if err := r.setLocationHeader(resp, true, desc, "/v2/"+rreq.Repo+"/blobs/"+rreq.Digest); err != nil { - return err - } - resp.WriteHeader(http.StatusCreated) - return nil -} - -func (r *registry) handleManifestPut(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - mediaType := req.Header.Get("Content-Type") - if mediaType == "" { - mediaType = mediaTypeOctetStream - } - // TODO check that the media type is valid? - // TODO size limit - data, err := io.ReadAll(req.Body) - if err != nil { - return fmt.Errorf("cannot read content: %v", err) - } - dig := ocidigest.FromBytes(data) - params := &oci.PushManifestParameters{} - if rreq.Tag != "" { - params.Tags = []string{rreq.Tag} - } else { - if rreq.Digest != dig.String() { - return oci.ErrDigestInvalid - } - params.Tags = rreq.Tags - } - subjectDesc, err := subjectFromManifest(req.Header.Get("Content-Type"), data) - if err != nil { - return fmt.Errorf("invalid manifest JSON: %v", err) - } - desc, err := r.backend.PushManifest(ctx, rreq.Repo, data, mediaType, params) - if err != nil { - return err - } - if err := r.setLocationHeader(resp, false, desc, "/v2/"+rreq.Repo+"/manifests/"+desc.Digest.String()); err != nil { - return err - } - if subjectDesc != nil { - resp.Header().Set("OCI-Subject", subjectDesc.Digest.String()) - } - // TODO OCI-Subject header? - resp.WriteHeader(http.StatusCreated) - return nil -} - -func subjectFromManifest(contentType string, data []byte) (*oci.Descriptor, error) { - switch contentType { - case oci.MediaTypeImageManifest, oci.MediaTypeDockerManifest, - oci.MediaTypeImageIndex, oci.MediaTypeDockerManifestList: - default: - return nil, nil - } - var m oci.IndexOrManifest - if err := json.Unmarshal(data, &m); err != nil { - return nil, err - } - if m.MediaType == "" { - m.MediaType = contentType - } - if err := m.Validate(); err != nil { - return nil, err - } - return m.Subject, nil -} - -func (r *registry) locationForUploadID(repo string, uploadID string) string { - _, loc := (&ocirequest.Request{ - Kind: ocirequest.ReqBlobUploadInfo, - Repo: repo, - UploadID: uploadID, - }).MustConstruct() - return loc -} - -func chunkRange(req *http.Request) (start, end int64, _ error) { - var rangeOK bool - if s := req.Header.Get("Content-Range"); s != "" { - start, end, rangeOK = ocirequest.ParseRange(s) - if !rangeOK { - return 0, 0, badAPIUseError("we don't understand your Content-Range") - } - } - - if rangeOK && req.ContentLength >= 0 { - rangeLength := end - start - if rangeLength != req.ContentLength { - return 0, 0, badAPIUseError("Content-Range implies a length of %d but Content-Length is %d", rangeLength, req.ContentLength) - } - } - - // The registry here is stateless, so it doesn't remember what minimum chunk size - // the backend registry suggested that we should use. - // We rely on the HTTP client to remember that minimum and use it, - // which would mean that each PATCH chunk before the last should be at least as large. - // Extract that size from either Content-Range or Content-Length; - // if neither is set, we fall back to 0, letting the backend assume a default. - if !rangeOK && req.ContentLength >= 0 { - end = req.ContentLength - } - return start, end, nil -}