diff --git a/.github/workflows/pages.yaml b/.github/workflows/pages.yaml new file mode 100644 index 00000000..65f1ac1b --- /dev/null +++ b/.github/workflows/pages.yaml @@ -0,0 +1,88 @@ +name: Publish documentation + +on: + push: + branches: [main] + paths: + - docs/** + - mkdocs.yml + - .github/workflows/pages.yaml + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Configure GitHub Pages + uses: actions/configure-pages@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install documentation builder + run: python -m pip install mkdocs-material==9.6.14 + + - name: Select end-user documentation + run: | + mkdir -p .pages-docs + cp docs/index.md .pages-docs/ + cp docs/quickstart.md .pages-docs/ + cp docs/getting-started.md .pages-docs/ + cp docs/commands.md .pages-docs/ + cp docs/pointer-files.md .pages-docs/ + cp docs/adding-s3-files.md .pages-docs/ + cp docs/remove-files.md .pages-docs/ + cp docs/troubleshooting.md .pages-docs/ + cp docs/*.png .pages-docs/ + + - name: Build documentation and presentation + run: | + mkdir -p .pages-build + npx --yes @marp-team/marp-cli@4.2.3 \ + docs/anvil-terra-poc-presentation.md \ + --html \ + --template bespoke \ + --output .pages-build/anvil-terra-poc-presentation.html + mkdocs build --strict + # Keep the standalone Bespoke document byte-for-byte intact. Passing + # it through MkDocs is unnecessary for this standalone entry point. + cp .pages-build/anvil-terra-poc-presentation.html \ + _site/anvil-terra-poc-presentation.html + + - name: Verify presentation artifact + run: | + # A Bespoke deck needs its embedded browser runtime for slide paging. + grep -q '` Example: ```bash -git drs remote add gen3 production HTAN_INT/BForePC --cred /path/to/credentials.json +git drs remote add production calypr --scope HTAN_INT/BForePC \ + --credential file:/path/to/credentials.json ``` Current command split: @@ -91,7 +95,8 @@ Push and pull depend on server-side bucket mapping for the requested scope. That | --- | --- | | `git drs install` | Install global `git-drs` filter config | | `git drs init` | Explicitly initialize or repair repository-local `git-drs` state | -| `git drs remote add gen3 [remote] ` | Add or refresh a Gen3/Syfon remote | +| `git drs remote add ` | Add a DRS remote (operational: Calypr/Gen3 and Terra; catalog-only: Synapse and CGC) | +| `git drs preset list` | List the non-secret presets embedded in this release | | `git drs remote list` | List configured remotes | | `git drs remote remove ` | Remove a configured DRS remote | | `git drs remote set ` | Set the default remote | @@ -108,10 +113,12 @@ Push and pull depend on server-side bucket mapping for the requested scope. That ## Documentation +- [Documentation Home](docs/index.md) - [Getting Started](docs/getting-started.md) - [Commands Reference](docs/commands.md) - [Troubleshooting](docs/troubleshooting.md) - [Developer Guide](docs/developer-guide.md) +- [Publishing documentation with GitHub Pages](docs/github-pages.md) - [GA4GH DRS Scalability Gaps](docs/ga4gh-drs-scalability-gaps.md) ## Requirements diff --git a/cmd/addref/add-ref.go b/cmd/addref/add-ref.go index 84f3b7f0..868ef673 100644 --- a/cmd/addref/add-ref.go +++ b/cmd/addref/add-ref.go @@ -2,24 +2,51 @@ package addref import ( "context" + "crypto/sha256" + "encoding/csv" + "encoding/hex" "fmt" + "net/url" "os" "path/filepath" + "sort" + "strconv" + "strings" "github.com/calypr/git-drs/internal/config" "github.com/calypr/git-drs/internal/drslog" + "github.com/calypr/git-drs/internal/drsobject" + "github.com/calypr/git-drs/internal/gitrepo" "github.com/calypr/git-drs/internal/lfs" "github.com/calypr/git-drs/internal/remoteruntime" + "github.com/calypr/git-drs/internal/resolver" + drsapi "github.com/calypr/syfon/apigen/client/drs" + syclient "github.com/calypr/syfon/client" + "github.com/calypr/syfon/client/hash" "github.com/spf13/cobra" ) var remote string +var remoteType string +var manifestPath string +var dryRun bool var Cmd = &cobra.Command{ - Use: "add-ref ", + Use: "add-ref | --manifest ", Short: "Add a reference to an existing DRS object via URI", Long: "Add a reference to an existing DRS object via URI. Requires that the sha256 of the file is already in the cache", - Args: cobra.ExactArgs(2), + Args: func(cmd *cobra.Command, args []string) error { + if manifestPath != "" && len(args) == 0 { + return nil + } + if manifestPath != "" { + return fmt.Errorf("positional arguments cannot be combined with --manifest") + } + return cobra.ExactArgs(2)(cmd, args) + }, RunE: func(cmd *cobra.Command, args []string) error { + if manifestPath != "" { + return runManifest(cmd, manifestPath) + } drsUri := args[0] dstPath := args[1] @@ -37,16 +64,34 @@ var Cmd = &cobra.Command{ logger.Error(fmt.Sprintf("Error getting remote: %v", err)) return err } + selected := cfg.GetRemote(remoteName) + if selected == nil { + return fmt.Errorf("remote %q is not configured", remoteName) + } + dstPath, err = safeDestination(dstPath) + if err != nil { + return err + } client, err := remoteruntime.New(cfg, remoteName, logger) if err != nil { return err } + if !client.CanResolve() { + return fmt.Errorf("remote %q cannot resolve DRS objects", remoteName) + } + if remoteType != "" && remoteType != string(client.RemoteType) { + return fmt.Errorf("--remote-type %q conflicts with configured remote %q type %q", remoteType, remoteName, client.RemoteType) + } - obj, err := client.Client.DRS().GetObject(context.Background(), drsUri) + obj, err := resolveAddRefObject(context.Background(), cfg, remoteName, client, drsUri) if err != nil { return err } + if dryRun { + _, err := fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%d\n1 reference(s) validated\n", drsUri, args[1], obj.Size) + return err + } dirPath := filepath.Dir(dstPath) _, err = os.Stat(dirPath) if os.IsNotExist(err) { @@ -54,11 +99,314 @@ var Cmd = &cobra.Command{ os.MkdirAll(dirPath, os.ModePerm) } - err = lfs.CreateLfsPointer(&obj, dstPath) - return err + if err := createAddRefPointer(&obj, dstPath, drsUri); err != nil { + return err + } + if _, err := gitrepo.TrackReadOnly(cmd.Context(), args[1]); err != nil { + return fmt.Errorf("track add-ref destination %s: %w", args[1], err) + } + if err := persistAddRefObject(&obj, drsUri, remoteName); err != nil { + return fmt.Errorf("write source DRS metadata: %w", err) + } + return nil }, } +func safeDestination(dst string) (string, error) { + if filepath.IsAbs(dst) { + return "", fmt.Errorf("destination path must be relative to the repository: %s", dst) + } + clean := filepath.Clean(dst) + if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("destination path escapes the repository: %s", dst) + } + root, err := gitrepo.GitTopLevel() + if err != nil { + return "", err + } + root, err = filepath.EvalSymlinks(root) + if err != nil { + return "", fmt.Errorf("resolve repository root: %w", err) + } + + // Do not allow an existing path component to redirect the eventual pointer + // write. Checking only the cleaned path is insufficient: os.WriteFile and + // MkdirAll follow symlinks, including a symlink at the destination itself. + current := root + for _, component := range strings.Split(clean, string(filepath.Separator)) { + current = filepath.Join(current, component) + info, statErr := os.Lstat(current) + if os.IsNotExist(statErr) { + break + } + if statErr != nil { + return "", fmt.Errorf("inspect destination path %s: %w", dst, statErr) + } + if info.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("destination path contains a symlink: %s", dst) + } + } + return filepath.Join(root, clean), nil +} + func init() { Cmd.Flags().StringVarP(&remote, "remote", "r", "", "target remote DRS server (default: default_remote)") + Cmd.Flags().StringVar(&remoteType, "remote-type", "", "resolver remote type for DRS references (for example: terra)") + Cmd.Flags().StringVar(&manifestPath, "manifest", "", "add references from a tab-separated manifest") + Cmd.Flags().BoolVar(&dryRun, "dry-run", false, "validate and report references without writing pointers") + _ = Cmd.Flags().MarkHidden("remote-type") + _ = Cmd.Flags().MarkDeprecated("remote-type", "remote behavior is selected by --remote configuration") +} + +type manifestEntry struct { + uri, path, sha256 string + size *int64 + object drsapi.DrsObject + destination string +} + +func runManifest(cmd *cobra.Command, filename string) error { + f, err := os.Open(filename) + if err != nil { + return fmt.Errorf("open manifest: %w", err) + } + defer f.Close() + r := csv.NewReader(f) + r.Comma = '\t' + r.FieldsPerRecord = -1 + r.TrimLeadingSpace = true + rows, err := r.ReadAll() + if err != nil { + return fmt.Errorf("parse manifest: %w", err) + } + if len(rows) < 2 { + return fmt.Errorf("manifest must contain a header and at least one reference") + } + columns := map[string]int{} + for i, name := range rows[0] { + columns[strings.ToLower(strings.TrimSpace(name))] = i + } + for _, required := range []string{"drs_uri", "path"} { + if _, ok := columns[required]; !ok { + return fmt.Errorf("manifest is missing required %q column", required) + } + } + value := func(row []string, name string) string { + i, ok := columns[name] + if !ok || i >= len(row) { + return "" + } + return strings.TrimSpace(row[i]) + } + entries := make([]manifestEntry, 0, len(rows)-1) + seen := map[string]int{} + var problems []string + for i, row := range rows[1:] { + e := manifestEntry{uri: value(row, "drs_uri"), path: value(row, "path"), sha256: strings.ToLower(strings.TrimPrefix(value(row, "sha256"), "sha256:"))} + if _, _, err := resolver.NormalizeDRSURI(e.uri); err != nil { + problems = append(problems, fmt.Sprintf("row %d: %v", i+2, err)) + } + clean := filepath.Clean(e.path) + if prior, ok := seen[clean]; ok { + problems = append(problems, fmt.Sprintf("row %d: duplicate path %q (also row %d)", i+2, e.path, prior)) + } else { + seen[clean] = i + 2 + } + e.destination, err = safeDestination(e.path) + if err != nil { + problems = append(problems, fmt.Sprintf("row %d: %v", i+2, err)) + } + if raw := value(row, "size"); raw != "" { + n, parseErr := strconv.ParseInt(raw, 10, 64) + if parseErr != nil || n < 0 { + problems = append(problems, fmt.Sprintf("row %d: invalid size %q", i+2, raw)) + } else { + e.size = &n + } + } + if e.sha256 != "" && (len(e.sha256) != 64 || strings.Trim(e.sha256, "0123456789abcdef") != "") { + problems = append(problems, fmt.Sprintf("row %d: invalid sha256", i+2)) + } + entries = append(entries, e) + } + if len(problems) > 0 { + return fmt.Errorf("manifest validation failed:\n- %s", strings.Join(problems, "\n- ")) + } + cfg, err := config.LoadConfig() + if err != nil { + return err + } + remoteName, err := cfg.GetRemoteOrDefault(remote) + if err != nil { + return err + } + runtime, err := remoteruntime.New(cfg, remoteName, drslog.GetLogger()) + if err != nil { + return err + } + for i := range entries { + obj, resolveErr := resolveAddRefObject(cmd.Context(), cfg, remoteName, runtime, entries[i].uri) + if resolveErr != nil { + problems = append(problems, fmt.Sprintf("row %d: %v", i+2, resolveErr)) + continue + } + entries[i].object = obj + if entries[i].size != nil && *entries[i].size != obj.Size { + problems = append(problems, fmt.Sprintf("row %d: asserted size %d does not match authoritative size %d", i+2, *entries[i].size, obj.Size)) + } + authSHA := drsobject.NormalizeChecksum(hash.ConvertDrsChecksumsToHashInfo(obj.Checksums).SHA256) + if entries[i].sha256 != "" && !strings.EqualFold(entries[i].sha256, authSHA) { + problems = append(problems, fmt.Sprintf("row %d: asserted sha256 does not match authoritative checksum", i+2)) + } + } + if len(problems) > 0 { + return fmt.Errorf("manifest validation failed:\n- %s", strings.Join(problems, "\n- ")) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].path < entries[j].path }) + for _, e := range entries { + if dryRun { + fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%d\n", e.uri, e.path, e.object.Size) + continue + } + if err := os.MkdirAll(filepath.Dir(e.destination), 0o755); err != nil { + return err + } + err = createAddRefPointer(&e.object, e.destination, e.uri) + if err != nil { + return err + } + if _, err := gitrepo.TrackReadOnly(cmd.Context(), e.path); err != nil { + return fmt.Errorf("track add-ref destination %s: %w", e.path, err) + } + if err := persistAddRefObject(&e.object, e.uri, remoteName); err != nil { + return fmt.Errorf("write source DRS metadata for %s: %w", e.path, err) + } + } + fmt.Fprintf(cmd.OutOrStdout(), "%d reference(s) %s\n", len(entries), map[bool]string{true: "validated", false: "added"}[dryRun]) + return nil +} + +func persistAddRefObject(obj *drsapi.DrsObject, sourceURI string, remoteName config.Remote) error { + if obj.SelfUri == "" { + obj.SelfUri = sourceURI + } + return drsobject.WriteObject(gitrepo.DRSObjectsPath, obj, addRefLocalOID(sourceURI, remoteName, obj)) +} + +// createAddRefPointer keeps the source authority in Git. The metadata written +// under .git/drs is only a local cache, so a checksum-only pointer would leave +// another clone unable to route hydration back to the authority that resolved +// the reference. +func createAddRefPointer(obj *drsapi.DrsObject, dst, sourceURI string) error { + return lfs.CreateDRSPointer(obj, dst, sourceURI) +} + +func addRefLocalOID(sourceURI string, remoteName config.Remote, obj *drsapi.DrsObject) string { + if obj != nil { + if sha := drsobject.NormalizeChecksum(hash.ConvertDrsChecksumsToHashInfo(obj.Checksums).SHA256); sha != "" { + return sha + } + } + return derivedSourceOID(sourceURI, string(remoteName)) +} + +func derivedSourceOID(sourceURI, remoteName string) string { + sum := sha256.Sum256([]byte("git-drs-source-ref:v1\nsource_uri=" + sourceURI + "\nremote=" + remoteName + "\n")) + return hex.EncodeToString(sum[:]) +} + +type drsObjectGetter interface { + GetObject(context.Context, string) (drsapi.DrsObject, error) +} + +func resolveAddRefObject(ctx context.Context, cfg *config.Config, primaryRemote config.Remote, primary *remoteruntime.GitContext, drsURI string) (drsapi.DrsObject, error) { + if primary != nil && primary.RemoteType == config.TerraServerType { + anvil, err := newAnVILResolver(ctx, primary.Endpoint) + if err != nil { + return drsapi.DrsObject{}, err + } + resolved, err := anvil.GetObject(ctx, drsURI) + if err != nil { + return drsapi.DrsObject{}, err + } + checksums := make([]drsapi.Checksum, 0, len(resolved.Checksums)) + for _, checksum := range resolved.Checksums { + checksums = append(checksums, drsapi.Checksum{Type: checksum.Type, Checksum: checksum.Checksum}) + } + var name *string + if resolved.Name != "" { + name = &resolved.Name + } + return drsapi.DrsObject{Id: resolved.ID, Name: name, Size: resolved.Size, SelfUri: resolved.DRSURI, Checksums: checksums}, nil + } + objectID, sourceEndpoint, ok := parseDRSURIForSource(drsURI) + if !ok { + return primary.Client.DRS().GetObject(ctx, drsURI) + } + if sourceRemote, found := configuredRemoteForDRSURIHost(cfg, drsURI); found { + if sourceRemote == primaryRemote { + return primary.Client.DRS().GetObject(ctx, objectID) + } + client, err := remoteruntime.New(cfg, sourceRemote, primary.Logger) + if err != nil { + return drsapi.DrsObject{}, err + } + return client.Client.DRS().GetObject(ctx, objectID) + } + getter, err := newSourceDRSGetter(sourceEndpoint) + if err != nil { + return drsapi.DrsObject{}, err + } + return getter.GetObject(ctx, objectID) +} + +var newAnVILResolver = func(ctx context.Context, endpoint string) (resolver.Resolver, error) { + return resolver.NewAnVIL(ctx, endpoint) +} + +func parseDRSURIForSource(drsURI string) (objectID string, endpoint string, ok bool) { + u, err := url.Parse(strings.TrimSpace(drsURI)) + if err != nil || !strings.EqualFold(u.Scheme, "drs") || strings.TrimSpace(u.Host) == "" { + return "", "", false + } + objectID = strings.TrimPrefix(u.EscapedPath(), "/") + if objectID == "" { + return "", "", false + } + return objectID, "https://" + u.Host, true +} + +func configuredRemoteForDRSURIHost(cfg *config.Config, drsURI string) (config.Remote, bool) { + u, err := url.Parse(strings.TrimSpace(drsURI)) + if err != nil || !strings.EqualFold(u.Scheme, "drs") || strings.TrimSpace(u.Host) == "" { + return "", false + } + for name, selected := range cfg.Remotes { + remote := config.Config{Remotes: map[config.Remote]config.RemoteSelect{name: selected}}.GetRemote(name) + if remote == nil { + continue + } + endpoint, err := url.Parse(remote.GetEndpoint()) + if err != nil { + continue + } + if strings.EqualFold(endpoint.Host, u.Host) { + return name, true + } + } + return "", false +} + +var newSourceDRSGetter = newAnonymousSourceDRSGetter + +func newAnonymousSourceDRSGetter(endpoint string) (drsObjectGetter, error) { + raw, err := syclient.New(endpoint) + if err != nil { + return nil, err + } + client, ok := raw.(*syclient.Client) + if !ok { + return nil, fmt.Errorf("unexpected syfon client type %T", raw) + } + return client.DRS(), nil } diff --git a/cmd/addref/add-ref_test.go b/cmd/addref/add-ref_test.go index b8087f8a..93022405 100644 --- a/cmd/addref/add-ref_test.go +++ b/cmd/addref/add-ref_test.go @@ -1,18 +1,30 @@ package addref import ( + "bytes" + "context" + "net/http" + "net/http/httptest" "os" + "os/exec" "path/filepath" + "strings" "testing" + "github.com/calypr/git-drs/internal/config" + "github.com/calypr/git-drs/internal/drslog" + "github.com/calypr/git-drs/internal/drsobject" + "github.com/calypr/git-drs/internal/gitrepo" "github.com/calypr/git-drs/internal/lfs" + "github.com/calypr/git-drs/internal/remoteruntime" drsapi "github.com/calypr/syfon/apigen/client/drs" + "github.com/spf13/cobra" ) func TestCreateLfsPointer(t *testing.T) { obj := &drsapi.DrsObject{ Size: 10, - Checksums: []drsapi.Checksum{{Type: "sha256", Checksum: "abc"}}, + Checksums: []drsapi.Checksum{{Type: "sha256", Checksum: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}, } path := filepath.Join(t.TempDir(), "pointer") if err := lfs.CreateLfsPointer(obj, path); err != nil { @@ -40,3 +52,359 @@ func TestCreateLfsPointer_NoSHA256(t *testing.T) { t.Fatalf("expected error for missing sha256") } } + +func TestSafeDestinationRejectsSymlinkedParent(t *testing.T) { + repo := t.TempDir() + outside := t.TempDir() + runGitCmd(t, repo, "init") + if err := os.Symlink(outside, filepath.Join(repo, "data")); err != nil { + t.Fatal(err) + } + + oldWD, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(repo); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + + if _, err := safeDestination(filepath.Join("data", "pointer")); err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("expected symlink destination to be rejected, got %v", err) + } +} + +func TestSafeDestinationRejectsSymlinkedFile(t *testing.T) { + repo := t.TempDir() + outside := filepath.Join(t.TempDir(), "outside") + runGitCmd(t, repo, "init") + if err := os.WriteFile(outside, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(repo, "pointer")); err != nil { + t.Fatal(err) + } + + oldWD, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(repo); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + + if _, err := safeDestination("pointer"); err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("expected symlink destination to be rejected, got %v", err) + } +} + +func TestAddRefLocalOIDUsesDerivedSourceWhenSHA256Missing(t *testing.T) { + obj := &drsapi.DrsObject{Checksums: []drsapi.Checksum{{Type: "md5", Checksum: "md5"}}} + oid := addRefLocalOID("drs://example.org/object-1", "source", obj) + if len(oid) != 64 { + t.Fatalf("expected sha256-shaped derived oid, got %q", oid) + } + if oid != derivedSourceOID("drs://example.org/object-1", "source") { + t.Fatalf("expected derived source oid, got %s", oid) + } +} + +func TestRunManifestPersistsResolvedObjectMetadata(t *testing.T) { + const checksum = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + const sourceURI = "drs://source.example.org/object-1" + var resolvedEndpoint, resolvedObjectID string + oldGetter := newSourceDRSGetter + newSourceDRSGetter = func(endpoint string) (drsObjectGetter, error) { + resolvedEndpoint = endpoint + return fakeDRSObjectGetter{obj: drsapi.DrsObject{ + Id: "object-1", + Size: 42, + Checksums: []drsapi.Checksum{{Type: "sha256", Checksum: checksum}}, + }, gotObjectID: &resolvedObjectID}, nil + } + t.Cleanup(func() { newSourceDRSGetter = oldGetter }) + + repo := t.TempDir() + runGitCmd(t, repo, "init") + runGitCmd(t, repo, "config", "drs.default-remote", "source") + runGitCmd(t, repo, "config", "drs.remote.source.type", "local") + runGitCmd(t, repo, "config", "drs.remote.source.endpoint", "https://primary.example.org") + manifest := filepath.Join(repo, "references.tsv") + if err := os.WriteFile(manifest, []byte("drs_uri\tpath\n"+sourceURI+"\tdata/reference\n"), 0o600); err != nil { + t.Fatal(err) + } + + oldWD, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(repo); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + oldRemote, oldDryRun := remote, dryRun + remote, dryRun = "", false + t.Cleanup(func() { remote, dryRun = oldRemote, oldDryRun }) + + cmd := &cobra.Command{} + cmd.SetOut(&bytes.Buffer{}) + if err := runManifest(cmd, manifest); err != nil { + t.Fatalf("runManifest: %v", err) + } + obj, err := drsobject.ReadObject(gitrepo.DRSObjectsPath, checksum) + if err != nil { + t.Fatalf("read persisted manifest object: %v", err) + } + if obj.Id != "object-1" || obj.SelfUri != sourceURI { + t.Fatalf("unexpected persisted object: %+v", obj) + } + if resolvedObjectID != "object-1" { + t.Fatalf("unexpected resolved object ID: %q", resolvedObjectID) + } + if resolvedEndpoint != "https://source.example.org" { + t.Fatalf("unexpected resolver endpoint: %q", resolvedEndpoint) + } +} + +func TestCreateDRSPointerPreservesSourceURI(t *testing.T) { + obj := &drsapi.DrsObject{Size: 42} + path := filepath.Join(t.TempDir(), "pointer") + if err := lfs.CreateDRSPointer(obj, path, "drs://example.org/object-1"); err != nil { + t.Fatalf("CreateDRSPointer error: %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read pointer: %v", err) + } + expected := "version https://calypr.github.io/spec/v1\noid drs://example.org/object-1\nsize 42\n" + if string(data) != expected { + t.Fatalf("pointer mismatch: expected %q, got %q", expected, string(data)) + } +} + +func TestAddRefPointerPreservesSourceURIWhenObjectHasSHA256(t *testing.T) { + obj := &drsapi.DrsObject{ + Size: 42, + Checksums: []drsapi.Checksum{{Type: "sha256", Checksum: strings.Repeat("a", 64)}}, + } + path := filepath.Join(t.TempDir(), "pointer") + if err := createAddRefPointer(obj, path, "drs://source.example/object-1"); err != nil { + t.Fatalf("createAddRefPointer error: %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read pointer: %v", err) + } + want := "version https://calypr.github.io/spec/v1\n" + + "oid drs://source.example/object-1\n" + + "size 42\n" + + "sha256 " + strings.Repeat("a", 64) + "\n" + if string(data) != want { + t.Fatalf("pointer mismatch:\n got: %q\nwant: %q", data, want) + } +} + +func TestAddRefTrackingMakesDRSPointerDiscoverable(t *testing.T) { + repo := t.TempDir() + runGitCmd(t, repo, "init") + runGitCmd(t, repo, "config", "user.email", "test@example.com") + runGitCmd(t, repo, "config", "user.name", "Test User") + + oldWD, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(repo); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + + const path = "population_descriptor.tsv" + obj := &drsapi.DrsObject{Size: 200184} + if err := lfs.CreateDRSPointer(obj, path, "drs://drs.anv0:v2_example"); err != nil { + t.Fatal(err) + } + if _, err := gitrepo.TrackReadOnly(context.Background(), path); err != nil { + t.Fatal(err) + } + runGitCmd(t, repo, "add", ".gitattributes", path) + + files, err := lfs.GetTrackedLfsFiles(drslog.NewNoOpLogger()) + if err != nil { + t.Fatal(err) + } + info, ok := files[path] + if !ok { + t.Fatalf("expected add-ref pointer to be discoverable, got %+v", files) + } + if info.OidType != "drs" || info.Oid != "//drs.anv0:v2_example" || info.Size != 200184 { + t.Fatalf("unexpected pointer inventory: %+v", info) + } +} + +func runGitCmd(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v failed: %v: %s", args, err, out) + } +} + +func TestResolveAddRefObjectUsesSourceAuthorityRemoteCredentials(t *testing.T) { + var primaryRequests int + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + primaryRequests++ + http.Error(w, "primary remote must not resolve source DRS URI", http.StatusTeapot) + })) + defer primary.Close() + + var sourceRequests int + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sourceRequests++ + if r.URL.Path != "/ga4gh/drs/v1/objects/object-1" { + t.Fatalf("unexpected source path: %s", r.URL.Path) + } + user, pass, ok := r.BasicAuth() + if !ok || user != "source-user" || pass != "source-pass" { + t.Fatalf("expected source basic auth credentials, got ok=%v user=%q pass=%q", ok, user, pass) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"object-1","self_uri":"drs://` + r.Host + `/object-1","size":42}`)) + })) + defer source.Close() + + cfg := &config.Config{ + DefaultRemote: "primary", + Remotes: map[config.Remote]config.RemoteSelect{ + "primary": {Local: &config.LocalRemote{BaseURL: primary.URL}}, + "source": {Local: &config.LocalRemote{BaseURL: source.URL, BasicUsername: "source-user", BasicPassword: "source-pass"}}, + }, + } + primaryCtx, err := remoteruntime.New(cfg, "primary", drslog.NewNoOpLogger()) + if err != nil { + t.Fatalf("create primary runtime: %v", err) + } + + obj, err := resolveAddRefObject(context.Background(), cfg, "primary", primaryCtx, "drs://"+strings.TrimPrefix(source.URL, "http://")+"/object-1") + if err != nil { + t.Fatalf("resolveAddRefObject: %v", err) + } + if obj.Id != "object-1" || obj.Size != 42 { + t.Fatalf("unexpected object: %+v", obj) + } + if sourceRequests != 1 { + t.Fatalf("expected one source request, got %d", sourceRequests) + } + if primaryRequests != 0 { + t.Fatalf("expected primary not to be contacted, got %d requests", primaryRequests) + } +} + +func TestResolveAddRefObjectIgnoresPrimaryEvenWhenPrimaryCanResolve(t *testing.T) { + var primaryRequests int + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + primaryRequests++ + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"primary-object","self_uri":"drs://primary/object-1","size":99}`)) + })) + defer primary.Close() + + var sourceRequests int + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sourceRequests++ + if r.URL.Path != "/ga4gh/drs/v1/objects/object-1" { + t.Fatalf("unexpected source path: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"source-object","self_uri":"drs://` + r.Host + `/object-1","size":42}`)) + })) + defer source.Close() + + cfg := &config.Config{ + DefaultRemote: "primary", + Remotes: map[config.Remote]config.RemoteSelect{ + "primary": {Local: &config.LocalRemote{BaseURL: primary.URL}}, + "source": {Local: &config.LocalRemote{BaseURL: source.URL}}, + }, + } + primaryCtx, err := remoteruntime.New(cfg, "primary", drslog.NewNoOpLogger()) + if err != nil { + t.Fatalf("create primary runtime: %v", err) + } + + obj, err := resolveAddRefObject(context.Background(), cfg, "primary", primaryCtx, "drs://"+strings.TrimPrefix(source.URL, "http://")+"/object-1") + if err != nil { + t.Fatalf("resolveAddRefObject: %v", err) + } + if obj.Id != "source-object" || obj.Size != 42 { + t.Fatalf("expected object from source DRS authority/resolver, got %+v", obj) + } + if sourceRequests != 1 { + t.Fatalf("expected one source request, got %d", sourceRequests) + } + if primaryRequests != 0 { + t.Fatalf("expected primary not to be contacted, got %d requests", primaryRequests) + } +} + +type fakeDRSObjectGetter struct { + gotObjectID *string + obj drsapi.DrsObject +} + +func (g fakeDRSObjectGetter) GetObject(_ context.Context, objectID string) (drsapi.DrsObject, error) { + *g.gotObjectID = objectID + return g.obj, nil +} + +func TestResolveAddRefObjectUsesSourceAuthorityWhenNoRemoteMatches(t *testing.T) { + var primaryRequests int + primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + primaryRequests++ + http.Error(w, "primary remote must not resolve source DRS URI", http.StatusTeapot) + })) + defer primary.Close() + + cfg := &config.Config{ + DefaultRemote: "primary", + Remotes: map[config.Remote]config.RemoteSelect{ + "primary": {Local: &config.LocalRemote{BaseURL: primary.URL}}, + }, + } + primaryCtx, err := remoteruntime.New(cfg, "primary", drslog.NewNoOpLogger()) + if err != nil { + t.Fatalf("create primary runtime: %v", err) + } + + var gotEndpoint string + var gotObjectID string + oldGetter := newSourceDRSGetter + newSourceDRSGetter = func(endpoint string) (drsObjectGetter, error) { + gotEndpoint = endpoint + return fakeDRSObjectGetter{ + gotObjectID: &gotObjectID, + obj: drsapi.DrsObject{Id: "source-object", Size: 42}, + }, nil + } + defer func() { newSourceDRSGetter = oldGetter }() + + obj, err := resolveAddRefObject(context.Background(), cfg, "primary", primaryCtx, "drs://source.example.org/object-1") + if err != nil { + t.Fatalf("resolveAddRefObject: %v", err) + } + if obj.Id != "source-object" || obj.Size != 42 { + t.Fatalf("expected object from source DRS authority/resolver, got %+v", obj) + } + if gotEndpoint != "https://source.example.org" { + t.Fatalf("expected source endpoint, got %q", gotEndpoint) + } + if gotObjectID != "object-1" { + t.Fatalf("expected source object ID, got %q", gotObjectID) + } + if primaryRequests != 0 { + t.Fatalf("expected primary not to be contacted, got %d requests", primaryRequests) + } +} diff --git a/cmd/addref/terra_addref_acceptance_test.go b/cmd/addref/terra_addref_acceptance_test.go new file mode 100644 index 00000000..21c366f9 --- /dev/null +++ b/cmd/addref/terra_addref_acceptance_test.go @@ -0,0 +1,9 @@ +package addref + +import "testing" + +func TestAcceptanceAddRefExposesRemoteTypeForTerraReferences(t *testing.T) { + if Cmd.Flags().Lookup("remote-type") == nil { + t.Fatalf("expected add-ref to expose --remote-type so Terra DRS references can select a Terra resolver explicitly") + } +} diff --git a/cmd/addurl/local_state.go b/cmd/addurl/local_state.go index d76d3a56..982bd7ae 100644 --- a/cmd/addurl/local_state.go +++ b/cmd/addurl/local_state.go @@ -21,9 +21,10 @@ import ( ) type addURLDrsFile struct { - Name string - Size int64 - Oid string + Name string + Size int64 + Oid string + ContentSHA256 string } func drsobjectBuilder(bucket, organization, project, storagePrefix string) drsobject.Builder { @@ -43,9 +44,18 @@ func writeAddURLDrsObject(builder drsobject.Builder, file addURLDrsFile, objectP drsObj.Size = file.Size } else { drsID := uuid.NewSHA1(drsobject.UUIDNamespace, []byte(fmt.Sprintf("%s:%s", builder.Project, drsobject.NormalizeOid(file.Oid)))).String() - drsObj, err = builder.Build(file.Name, file.Oid, file.Size, drsID) - if err != nil { - return nil, fmt.Errorf("error building DRS object for oid %s: %w", file.Oid, err) + if file.ContentSHA256 != "" { + drsObj, err = builder.Build(file.Name, file.ContentSHA256, file.Size, drsID) + if err != nil { + return nil, fmt.Errorf("error building DRS object for oid %s: %w", file.Oid, err) + } + } else { + drsObj = &drsapi.DrsObject{ + Id: drsID, + SelfUri: objectPath, + Size: file.Size, + Name: &file.Name, + } } } diff --git a/cmd/addurl/main_test.go b/cmd/addurl/main_test.go index 175b85d8..0c4ebee8 100644 --- a/cmd/addurl/main_test.go +++ b/cmd/addurl/main_test.go @@ -143,6 +143,9 @@ func TestRunAddURL_WritesPointerAndLFSObject(t *testing.T) { if got := (*drsObject.AccessMethods)[0].AccessUrl.Url; got != "s3://bucket/path/to/file.bin" { t.Fatalf("unexpected access URL: %s", got) } + if len(drsObject.Checksums) != 0 { + t.Fatalf("expected unknown sha256 add-url metadata not to fabricate checksums, got %+v", drsObject.Checksums) + } } func TestPlaceholderOIDForUnknownSHA(t *testing.T) { diff --git a/cmd/addurl/run.go b/cmd/addurl/run.go index 4c440557..572beb0c 100644 --- a/cmd/addurl/run.go +++ b/cmd/addurl/run.go @@ -137,9 +137,10 @@ func (s *AddURLService) Run(cmd *cobra.Command, args []string) error { builder := drsobjectBuilder(scope.Bucket, org, project, scope.Prefix) file := addURLDrsFile{ - Name: input.path, - Size: objectInfo.SizeBytes, - Oid: oid, + Name: input.path, + Size: objectInfo.SizeBytes, + Oid: oid, + ContentSHA256: input.sha256, } if _, err := writeAddURLDrsObject(builder, file, input.objectURL); err != nil { return fmt.Errorf("write local DRS object: %w", err) diff --git a/cmd/filter/main.go b/cmd/filter/main.go index cc70f45b..a60a6da8 100644 --- a/cmd/filter/main.go +++ b/cmd/filter/main.go @@ -23,6 +23,7 @@ import ( internalfilter "github.com/calypr/git-drs/internal/filter" "github.com/calypr/git-drs/internal/lfs" "github.com/calypr/git-drs/internal/remoteruntime" + "github.com/calypr/git-drs/internal/resolver" internaltransfer "github.com/calypr/git-drs/internal/transfer" "github.com/spf13/cobra" ) @@ -52,6 +53,7 @@ func runFilter(cmd *cobra.Command, _ []string) error { } var drsCtx *remoteruntime.GitContext + var terraResolver resolver.Resolver remote, err := cfg.GetDefaultRemote() if err != nil { @@ -60,6 +62,11 @@ func runFilter(cmd *cobra.Command, _ []string) error { drsCtx, err = remoteruntime.New(cfg, remote, logger) if err != nil { logger.Info("DRS server not configured or unreachable", "err", err) + } else if drsCtx.RemoteType == config.TerraServerType && !internalfilter.ShouldSkipSmudge() { + terraResolver, err = resolver.NewAnVIL(ctx, drsCtx.Endpoint) + if err != nil { + return fmt.Errorf("filter: create Terra resolver: %w", err) + } } } @@ -70,7 +77,7 @@ func runFilter(cmd *cobra.Command, _ []string) error { logger.Debug("Resolved LFS root directory", "lfsRoot", lfsRoot) // Build the filter and register handlers. f := internalfilter.NewGitFilter(os.Stdin, os.Stdout, logger). - OnSmudge(makeSmudgeHandler(drsCtx, logger)). + OnSmudge(makeSmudgeHandler(drsCtx, terraResolver, logger)). OnClean(makeCleanHandler(lfsRoot, logger)) return f.Run(ctx) @@ -80,12 +87,15 @@ func runFilter(cmd *cobra.Command, _ []string) error { // Smudge handler — checkout: LFS pointer → real file content // -------------------------------------------------------------------------- -func makeSmudgeHandler(drsCtx *remoteruntime.GitContext, logger *slog.Logger) internalfilter.SmudgeFunc { +func makeSmudgeHandler(drsCtx *remoteruntime.GitContext, terraResolver resolver.Resolver, logger *slog.Logger) internalfilter.SmudgeFunc { return func(ctx context.Context, req internalfilter.FilterRequest, ptr io.Reader, dst io.Writer) error { logger.Debug("smudge handler invoked", "pathname", req.Pathname) var downloadFn internalfilter.SmudgeDownloadFunc if drsCtx != nil && !internalfilter.ShouldSkipSmudge() { downloadFn = func(callCtx context.Context, oid, cachePath string) error { + if terraResolver != nil { + return resolver.DownloadToCache(callCtx, terraResolver, normalizeDRSOID(oid), cachePath) + } return internaltransfer.DownloadToCachePath(callCtx, drsCtx, oid, cachePath) } } @@ -93,6 +103,13 @@ func makeSmudgeHandler(drsCtx *remoteruntime.GitContext, logger *slog.Logger) in } } +func normalizeDRSOID(oid string) string { + if len(oid) >= 2 && oid[:2] == "//" { + return "drs:" + oid + } + return oid +} + // -------------------------------------------------------------------------- // Clean handler — stage: real file content → LFS pointer // -------------------------------------------------------------------------- diff --git a/cmd/filter/main_test.go b/cmd/filter/main_test.go new file mode 100644 index 00000000..2cdccb72 --- /dev/null +++ b/cmd/filter/main_test.go @@ -0,0 +1,71 @@ +package filter + +import ( + "bytes" + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + internalfilter "github.com/calypr/git-drs/internal/filter" + "github.com/calypr/git-drs/internal/remoteruntime" + "github.com/calypr/git-drs/internal/resolver" +) + +type recordingResolver struct { + accessURL string + drsURI string +} + +func (r *recordingResolver) GetObject(_ context.Context, drsURI string) (*resolver.ResolvedObject, error) { + r.drsURI = drsURI + return &resolver.ResolvedObject{ + Size: 13, + AccessMethods: []resolver.AccessMethod{{AccessID: "access-1"}}, + }, nil +} + +func (r *recordingResolver) GetAccess(context.Context, string, string) (*resolver.ResolvedAccess, error) { + return &resolver.ResolvedAccess{URL: r.accessURL}, nil +} + +func TestSmudgeHandlerUsesTerraResolver(t *testing.T) { + t.Setenv("GIT_DRS_SKIP_SMUDGE", "false") + payload := []byte("terra payload") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(payload) + })) + defer server.Close() + + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git", "lfs", "objects"), 0o755); err != nil { + t.Fatal(err) + } + oldWD, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(repo); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + + r := &recordingResolver{accessURL: server.URL} + ctx := &remoteruntime.GitContext{} + handler := makeSmudgeHandler(ctx, r, slog.New(slog.NewTextHandler(io.Discard, nil))) + pointer := "version https://calypr.github.io/spec/v1\noid drs://example.org/object-1\nsize 13\n" + var output bytes.Buffer + if err := handler(context.Background(), internalfilter.FilterRequest{Pathname: "data.bin"}, bytes.NewBufferString(pointer), &output); err != nil { + t.Fatalf("smudge handler: %v", err) + } + if got, want := output.String(), string(payload); got != want { + t.Fatalf("smudged content = %q, want %q", got, want) + } + if got, want := r.drsURI, "drs://example.org/object-1"; got != want { + t.Fatalf("resolver DRS URI = %q, want %q", got, want) + } +} diff --git a/cmd/ping/main.go b/cmd/ping/main.go index 3069018a..d17a4760 100644 --- a/cmd/ping/main.go +++ b/cmd/ping/main.go @@ -3,7 +3,10 @@ package ping import ( "context" "fmt" + "io" "log/slog" + "net/http" + "net/url" "strings" "github.com/calypr/git-drs/internal/config" @@ -27,8 +30,16 @@ type statusInfo struct { AuthMode string } -var pingHealth = func(ctx context.Context, gc *remoteruntime.GitContext) error { - return gc.Client.Health().Ping(ctx) +type healthInfo struct { + ServiceInfo string +} + +var pingHealth = func(ctx context.Context, gc *remoteruntime.GitContext) (healthInfo, error) { + if gc != nil && gc.IsReadOnly() { + serviceInfo, err := pingTerraServiceInfo(ctx, gc.Endpoint) + return healthInfo{ServiceInfo: serviceInfo}, err + } + return healthInfo{}, gc.Client.Health().Ping(ctx) } var pingScopeAccess = func(ctx context.Context, gc *remoteruntime.GitContext) (scopeAccessInfo, error) { @@ -59,10 +70,14 @@ var Cmd = &cobra.Command{ } printStatus(status) - if err := pingHealth(cmd.Context(), gc); err != nil { + health, err := pingHealth(cmd.Context(), gc) + if err != nil { return fmt.Errorf("remote health check failed for %q (%s): %w", status.Remote, status.Endpoint, err) } fmt.Println("health: ok") + if strings.TrimSpace(health.ServiceInfo) != "" { + fmt.Printf("service-info: %s\n", health.ServiceInfo) + } scopeInfo, err := pingScopeAccess(cmd.Context(), gc) if err != nil { @@ -121,14 +136,7 @@ func resolveStatus(args []string, logger *slog.Logger) (statusInfo, *remoterunti Bucket: gc.BucketName, StoragePrefix: gc.StoragePrefix, AuthMode: authMode(gc), - } - switch remoteCfg.(type) { - case *config.Gen3Remote: - status.RemoteType = string(config.Gen3ServerType) - case *config.LocalRemote: - status.RemoteType = string(config.LocalServerType) - default: - status.RemoteType = "unknown" + RemoteType: string(gc.RemoteType), } return status, gc, nil @@ -171,11 +179,10 @@ func blankIfEmpty(v string) string { } func checkScopeAccess(ctx context.Context, gc *remoteruntime.GitContext) (scopeAccessInfo, error) { - if gc == nil || gc.Client == nil { + info := scopeAccessInfo{} + if gc == nil { return scopeAccessInfo{}, fmt.Errorf("DRS client unavailable") } - - info := scopeAccessInfo{} organization := strings.TrimSpace(gc.Organization) project := strings.TrimSpace(gc.ProjectId) bucket := strings.TrimSpace(gc.BucketName) @@ -183,6 +190,9 @@ func checkScopeAccess(ctx context.Context, gc *remoteruntime.GitContext) (scopeA if organization == "" && project == "" && bucket == "" { return info, nil } + if gc.Client == nil { + return scopeAccessInfo{}, fmt.Errorf("DRS client unavailable") + } info.Checked = true if organization != "" && project != "" { @@ -211,6 +221,51 @@ func checkScopeAccess(ctx context.Context, gc *remoteruntime.GitContext) (scopeA return info, nil } +func pingTerraServiceInfo(ctx context.Context, endpoint string) (string, error) { + if ctx == nil { + ctx = context.Background() + } + serviceInfoURL, err := terraServiceInfoURL(endpoint) + if err != nil { + return "", err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, serviceInfoURL, nil) + if err != nil { + return "", err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + serviceInfo, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return "", fmt.Errorf("terra DRS service-info returned %s", resp.Status) + } + if readErr != nil { + return "", readErr + } + return strings.TrimSpace(string(serviceInfo)), nil +} + +func terraServiceInfoURL(endpoint string) (string, error) { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return "", fmt.Errorf("terra endpoint is empty") + } + parsed, err := url.Parse(endpoint) + if err != nil { + return "", err + } + if parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("terra endpoint must be an absolute URL: %q", endpoint) + } + parsed.Path = strings.TrimRight(parsed.Path, "/") + "/ga4gh/drs/v1/service-info" + parsed.RawQuery = "" + parsed.Fragment = "" + return parsed.String(), nil +} + func visibleBucketForScope(ctx context.Context, gc *remoteruntime.GitContext, organization, project string) (string, error) { payload, err := gc.Client.Buckets().List(ctx) if err != nil { diff --git a/cmd/ping/main_test.go b/cmd/ping/main_test.go index eb1c8141..19bd8f8a 100644 --- a/cmd/ping/main_test.go +++ b/cmd/ping/main_test.go @@ -5,6 +5,8 @@ import ( "context" "errors" "io" + "net/http" + "net/http/httptest" "os" "strings" "testing" @@ -28,6 +30,67 @@ func TestPingCmdArgs(t *testing.T) { } } +func TestAcceptancePingTerraDRSServer(t *testing.T) { + var serviceInfoRequests int + terraDRS := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ga4gh/drs/v1/service-info" { + t.Fatalf("expected Terra DRS service-info ping, got %s %s", r.Method, r.URL.Path) + } + serviceInfoRequests++ + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"terra-drs","name":"Terra DRS","type":{"group":"org.ga4gh","artifact":"drs","version":"1.0.0"}}`)) + })) + t.Cleanup(terraDRS.Close) + + tmpDir := testutils.SetupTestGitRepo(t) + testutils.CreateTestConfig(t, tmpDir, &config.Config{ + DefaultRemote: config.Remote("anvil"), + Remotes: map[config.Remote]config.RemoteSelect{ + config.Remote("anvil"): { + Terra: &config.TerraRemote{ + Endpoint: terraDRS.URL, + Auth: "google-adc", + Mode: "read-only", + }, + }, + }, + }) + + oldStdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + os.Stdout = w + t.Cleanup(func() { os.Stdout = oldStdout }) + + runErr := Cmd.RunE(Cmd, []string{"anvil"}) + _ = w.Close() + if runErr != nil { + t.Fatalf("Cmd.RunE returned error: %v", runErr) + } + + var buf bytes.Buffer + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("read stdout: %v", err) + } + got := buf.String() + for _, want := range []string{ + "remote: anvil (default)", + "type: terra", + "endpoint: " + terraDRS.URL, + "health: ok", + `service-info: {"id":"terra-drs","name":"Terra DRS","type":{"group":"org.ga4gh","artifact":"drs","version":"1.0.0"}}`, + } { + if !strings.Contains(got, want) { + t.Fatalf("expected output to contain %q, got %q", want, got) + } + } + if serviceInfoRequests != 1 { + t.Fatalf("expected exactly one Terra DRS service-info ping, got %d", serviceInfoRequests) + } +} + func TestResolveStatusLocalRemote(t *testing.T) { tmpDir := testutils.SetupTestGitRepo(t) testutils.CreateTestConfig(t, tmpDir, &config.Config{ @@ -90,11 +153,11 @@ func TestPingRunEPrintsStatusAndHealth(t *testing.T) { } oldHealth := pingHealth - pingHealth = func(ctx context.Context, gc *remoteruntime.GitContext) error { + pingHealth = func(ctx context.Context, gc *remoteruntime.GitContext) (healthInfo, error) { if gc == nil || gc.ProjectId != "end_to_end_test" { t.Fatalf("unexpected git context: %+v", gc) } - return nil + return healthInfo{}, nil } t.Cleanup(func() { pingHealth = oldHealth }) @@ -149,6 +212,54 @@ func TestPingRunEPrintsStatusAndHealth(t *testing.T) { } } +func TestPingRunEPrintsServiceInfo(t *testing.T) { + tmpDir := testutils.SetupTestGitRepo(t) + testutils.CreateTestConfig(t, tmpDir, &config.Config{ + DefaultRemote: config.Remote(config.ORIGIN), + Remotes: map[config.Remote]config.RemoteSelect{ + config.Remote(config.ORIGIN): { + Local: &config.LocalRemote{ + BaseURL: "http://127.0.0.1:8080", + }, + }, + }, + }) + + oldHealth := pingHealth + pingHealth = func(ctx context.Context, gc *remoteruntime.GitContext) (healthInfo, error) { + return healthInfo{ServiceInfo: `{"id":"terra-drs","name":"Terra DRS"}`}, nil + } + t.Cleanup(func() { pingHealth = oldHealth }) + + oldScopeAccess := pingScopeAccess + pingScopeAccess = func(ctx context.Context, gc *remoteruntime.GitContext) (scopeAccessInfo, error) { + return scopeAccessInfo{}, nil + } + t.Cleanup(func() { pingScopeAccess = oldScopeAccess }) + + oldStdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + os.Stdout = w + t.Cleanup(func() { os.Stdout = oldStdout }) + + runErr := Cmd.RunE(Cmd, nil) + _ = w.Close() + if runErr != nil { + t.Fatalf("Cmd.RunE returned error: %v", runErr) + } + + var buf bytes.Buffer + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("read stdout: %v", err) + } + if got, want := buf.String(), `service-info: {"id":"terra-drs","name":"Terra DRS"}`; !strings.Contains(got, want) { + t.Fatalf("expected output to contain %q, got %q", want, got) + } +} + func TestPingRunEReturnsReadableScopeError(t *testing.T) { tmpDir := testutils.SetupTestGitRepo(t) testutils.CreateTestConfig(t, tmpDir, &config.Config{ @@ -169,7 +280,7 @@ func TestPingRunEReturnsReadableScopeError(t *testing.T) { } oldHealth := pingHealth - pingHealth = func(ctx context.Context, gc *remoteruntime.GitContext) error { return nil } + pingHealth = func(ctx context.Context, gc *remoteruntime.GitContext) (healthInfo, error) { return healthInfo{}, nil } t.Cleanup(func() { pingHealth = oldHealth }) oldScopeAccess := pingScopeAccess diff --git a/cmd/ping/terra_integration_test.go b/cmd/ping/terra_integration_test.go new file mode 100644 index 00000000..6321dded --- /dev/null +++ b/cmd/ping/terra_integration_test.go @@ -0,0 +1,24 @@ +//go:build integration + +package ping + +import ( + "context" + "os" + "testing" + "time" +) + +func TestIntegrationPingTerraDRSServer(t *testing.T) { + endpoint := os.Getenv("GIT_DRS_TERRA_DRS_ENDPOINT") + if endpoint == "" { + endpoint = "https://data.terra.bio" + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + t.Cleanup(cancel) + + if _, err := pingTerraServiceInfo(ctx, endpoint); err != nil { + t.Fatalf("ping Terra DRS service-info endpoint %q: %v", endpoint, err) + } +} diff --git a/cmd/preset/root.go b/cmd/preset/root.go new file mode 100644 index 00000000..47cca59c --- /dev/null +++ b/cmd/preset/root.go @@ -0,0 +1,40 @@ +package preset + +import ( + "fmt" + "sort" + + "github.com/calypr/git-drs/internal/presets" + "github.com/spf13/cobra" +) + +var Cmd = &cobra.Command{Use: "preset", Short: "Inspect built-in DRS server presets"} + +var listCmd = &cobra.Command{Use: "list", Short: "List built-in presets", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + items, err := presets.List() + if err != nil { + return err + } + sort.Slice(items, func(i, j int) bool { return items[i].Alias < items[j].Alias }) + for _, p := range items { + fmt.Fprintf(cmd.OutOrStdout(), "%-10s %-8s %-30s %s built-in v%d\n", p.Alias, p.Provider, p.Auth, p.Endpoint, presets.CatalogVersion) + } + return nil +}} + +var showCmd = &cobra.Command{Use: "show ", Short: "Show a built-in preset", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + p, ok, err := presets.Lookup(args[0]) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("preset %q not found", args[0]) + } + fmt.Fprintf(cmd.OutOrStdout(), "alias: %s\nendpoint: %s\nprovider: %s\nauth: %s\nsource: built-in\ncatalog-version: %d\n", p.Alias, p.Endpoint, p.Provider, p.Auth, presets.CatalogVersion) + if p.RegistryServiceID != "" { + fmt.Fprintf(cmd.OutOrStdout(), "registry-service-id: %s\n", p.RegistryServiceID) + } + return nil +}} + +func init() { Cmd.AddCommand(listCmd, showCmd) } diff --git a/cmd/preset/root_test.go b/cmd/preset/root_test.go new file mode 100644 index 00000000..df7019c2 --- /dev/null +++ b/cmd/preset/root_test.go @@ -0,0 +1,37 @@ +package preset + +import ( + "bytes" + "strings" + "testing" +) + +func TestListIncludesAuthenticationType(t *testing.T) { + var out bytes.Buffer + listCmd.SetOut(&out) + t.Cleanup(func() { listCmd.SetOut(nil) }) + + if err := listCmd.RunE(listCmd, nil); err != nil { + t.Fatal(err) + } + + for _, want := range []string{ + "calypr gen3 provider-helper:gen3-profile", + "cgc cgc bearer", + "synapse synapse bearer", + "terra terra google-adc", + } { + if fields := strings.Fields(out.String()); !containsConsecutive(fields, strings.Fields(want)) { + t.Errorf("preset list output does not include %q:\n%s", want, out.String()) + } + } +} + +func containsConsecutive(values, want []string) bool { + for i := 0; i+len(want) <= len(values); i++ { + if strings.Join(values[i:i+len(want)], " ") == strings.Join(want, " ") { + return true + } + } + return false +} diff --git a/cmd/pull/main.go b/cmd/pull/main.go index 0b2a848e..b69e41b8 100644 --- a/cmd/pull/main.go +++ b/cmd/pull/main.go @@ -21,6 +21,7 @@ import ( "github.com/calypr/git-drs/internal/lfs" "github.com/calypr/git-drs/internal/lookup" "github.com/calypr/git-drs/internal/remoteruntime" + "github.com/calypr/git-drs/internal/resolver" internaltransfer "github.com/calypr/git-drs/internal/transfer" drsapi "github.com/calypr/syfon/apigen/client/drs" sycommon "github.com/calypr/syfon/client/common" @@ -93,6 +94,16 @@ var Cmd = &cobra.Command{ logg.Error(fmt.Sprintf("error creating DRS client: %s", err)) return err } + var anvil resolver.Resolver + if !drsCtx.CanDownload() || !drsCtx.CanResolve() { + return fmt.Errorf("remote %q does not support resolving and downloading DRS objects", remote) + } + if drsCtx.IsReadOnly() { + anvil, err = resolver.NewAnVIL(cmd.Context(), drsCtx.Endpoint) + if err != nil { + return err + } + } progress := internaltransfer.NewPullProgressRenderer(os.Stderr) progress.OnPlan(toPullFiles(pointers)) @@ -110,7 +121,7 @@ var Cmd = &cobra.Command{ if err != nil { return fmt.Errorf("failed to resolve LFS object path for %s: %w", f.Oid, err) } - state, err := inspectCachedObject(cachePath, f.Oid, f.Size) + state, err := inspectCachedPointer(cachePath, f) if err == nil && state.complete { continue } else if err != nil { @@ -126,6 +137,9 @@ var Cmd = &cobra.Command{ if len(missingOIDs) > 0 { prefetched := make(map[string]drsapi.DrsObject, len(missingOIDs)) for _, oid := range missingOIDs { + if isDRSPointerOID(oid) { + continue + } recs, err := lookup.ObjectsByHashForScope(ctx, drsCtx, oid) if err != nil || len(recs) == 0 { continue @@ -156,7 +170,7 @@ var Cmd = &cobra.Command{ if err != nil { return fmt.Errorf("failed to resolve LFS object path for %s: %w", f.Oid, err) } - state, err := inspectCachedObject(dstPath, f.Oid, f.Size) + state, err := inspectCachedPointer(dstPath, f) if err == nil && state.complete { continue } else if err != nil { @@ -183,11 +197,22 @@ var Cmd = &cobra.Command{ continue } } - if err := internaltransfer.DownloadToCachePath(downloadCtx, drsCtx, f.Oid, dstPath); err != nil { + if isDRSPointerOID(f.Oid) { + var downloadErr error + if anvil != nil { + downloadErr = resolver.DownloadToCache(downloadCtx, anvil, normalizeDRSPointerOID(f.Oid), dstPath) + } else { + downloadErr = internaltransfer.DownloadDRSURIToCachePath(downloadCtx, drsCtx, f.Oid, dstPath) + } + if downloadErr != nil { + debugCtx := buildPullDownloadDebugContext(ctx, drsCtx, f.Oid) + return fmt.Errorf("failed to download DRS URI %s to %s: %w\npull-debug: %s", f.Oid, dstPath, downloadErr, debugCtx) + } + } else if err := internaltransfer.DownloadToCachePath(downloadCtx, drsCtx, f.Oid, dstPath); err != nil { debugCtx := buildPullDownloadDebugContext(ctx, drsCtx, f.Oid) return fmt.Errorf("failed to download oid %s to %s: %w\npull-debug: %s", f.Oid, dstPath, err, debugCtx) } - if err := verifyObjectAtPath(dstPath, f.Oid, f.Size); err != nil { + if err := verifyPointerAtPath(dstPath, f); err != nil { _ = os.Remove(dstPath) return fmt.Errorf("downloaded invalid cached object for oid %s: %w", f.Oid, err) } @@ -196,7 +221,8 @@ var Cmd = &cobra.Command{ logg.Debug("no missing pointer objects to download") } - if err := checkoutDownloadedFiles(pointers, progress); err != nil { + readOnly := drsCtx.IsReadOnly() + if err := checkoutDownloadedFiles(pointers, progress, readOnly); err != nil { return err } if err := refreshGitIndexForHydratedFiles(pointers); err != nil { @@ -207,10 +233,18 @@ var Cmd = &cobra.Command{ }, } +func normalizeDRSPointerOID(oid string) string { + if strings.HasPrefix(oid, "//") { + return "drs:" + oid + } + return oid +} + type pointerFile struct { - Name string - Oid string - Size int64 + Name string + Oid string + Size int64 + SHA256 string } func collectPointerFiles(inventory map[string]lfs.LfsFileInfo, patterns []string) []pointerFile { @@ -226,11 +260,46 @@ func collectPointerFiles(inventory map[string]lfs.LfsFileInfo, patterns []string files := make([]pointerFile, 0, len(keys)) for _, path := range keys { info := inventory[path] - files = append(files, pointerFile{Name: path, Oid: info.Oid, Size: info.Size}) + files = append(files, pointerFile{Name: path, Oid: info.Oid, Size: info.Size, SHA256: info.SHA256}) } return files } +func inspectCachedPointer(path string, file pointerFile) (cachedObjectState, error) { + state, err := inspectCachedObject(path, file.Oid, file.Size) + if err != nil || !state.complete || file.SHA256 == "" { + return state, err + } + actual, err := calculateFileSHA256(path) + if err != nil { + return state, err + } + state.complete = strings.EqualFold(actual, file.SHA256) + return state, nil +} + +func verifyPointerAtPath(path string, file pointerFile) error { + if err := verifyObjectAtPath(path, file.Oid, file.Size); err != nil { + return err + } + if file.SHA256 == "" { + return nil + } + actual, err := calculateFileSHA256(path) + if err != nil { + return err + } + if !strings.EqualFold(actual, file.SHA256) { + return fmt.Errorf("sha256 mismatch: expected %s, got %s", file.SHA256, actual) + } + return nil +} + +func isDRSPointerOID(oid string) bool { + oid = strings.TrimSpace(oid) + return strings.HasPrefix(oid, "//") || strings.HasPrefix(strings.ToLower(oid), "drs://") +} + func progressContextForPointer(ctx context.Context, progress *internaltransfer.PullProgressRenderer, file pointerFile) context.Context { ctx = sycommon.WithOid(ctx, file.Name) return sycommon.WithProgress(ctx, func(ev sycommon.ProgressEvent) error { @@ -295,7 +364,7 @@ func inspectCachedObject(path, expectedOID string, expectedSize int64) (cachedOb if expectedSize <= 0 && info.Size() <= 0 { return state, nil } - if strings.TrimSpace(expectedOID) == "" { + if strings.TrimSpace(expectedOID) == "" || strings.HasPrefix(strings.TrimSpace(expectedOID), "//") || strings.HasPrefix(strings.ToLower(strings.TrimSpace(expectedOID)), "drs://") { state.complete = true return state, nil } @@ -374,7 +443,7 @@ func globToRegexp(pattern string) string { return b.String() } -func checkoutDownloadedFiles(files []pointerFile, progress *internaltransfer.PullProgressRenderer) error { +func checkoutDownloadedFiles(files []pointerFile, progress *internaltransfer.PullProgressRenderer, readOnly bool) error { for _, f := range files { if strings.TrimSpace(f.Name) == "" || strings.TrimSpace(f.Oid) == "" { continue @@ -397,6 +466,17 @@ func checkoutDownloadedFiles(files []pointerFile, progress *internaltransfer.Pul return fmt.Errorf("failed to create directory for %s: %w", f.Name, err) } } + // A previous pull may have made this path read-only. Temporarily restore + // owner write permission so that a later pull can safely replace it. + if info, statErr := os.Stat(f.Name); statErr == nil { + if err := os.Chmod(f.Name, info.Mode().Perm()|0o200); err != nil { + src.Close() + return fmt.Errorf("failed to make %s writable for checkout: %w", f.Name, err) + } + } else if !os.IsNotExist(statErr) { + src.Close() + return fmt.Errorf("failed to inspect checkout path %s: %w", f.Name, statErr) + } dst, err := os.OpenFile(f.Name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) if err != nil { src.Close() @@ -420,6 +500,11 @@ func checkoutDownloadedFiles(files []pointerFile, progress *internaltransfer.Pul } return fmt.Errorf("checked out invalid content for %s: %w", f.Name, err) } + if readOnly { + if err := os.Chmod(f.Name, 0o444); err != nil { + return fmt.Errorf("failed to make pulled file %s read-only: %w", f.Name, err) + } + } progress.OnCompleted(toPullFile(f)) } return nil @@ -461,6 +546,12 @@ func refreshGitIndexForHydratedFiles(files []pointerFile) error { } func buildPullDownloadDebugContext(ctx context.Context, drsCtx *remoteruntime.GitContext, oid string) string { + if drsCtx == nil { + return fmt.Sprintf("oid=%s resolver=unavailable", oid) + } + if drsCtx.Client == nil { + return fmt.Sprintf("oid=%s resolver=%s", oid, drsCtx.RemoteType) + } recs, err := lookup.ObjectsByHashForScope(ctx, drsCtx, oid) if err != nil { return fmt.Sprintf("oid=%s query_error=%v", oid, err) @@ -489,7 +580,7 @@ func buildPullDownloadDebugContext(ctx context.Context, drsCtx *remoteruntime.Gi if am.AccessId != nil { accessID = strings.TrimSpace(*am.AccessId) } - methods = append(methods, fmt.Sprintf("{type=%s access_id=%s url_scheme=%s url=%s}", am.Type, accessID, scheme, rawURL)) + methods = append(methods, fmt.Sprintf("{type=%s access_id=%s url_scheme=%s}", am.Type, accessID, scheme)) } } return fmt.Sprintf("oid=%s did=%s size=%d access_methods=%s", oid, strings.TrimSpace(match.Id), match.Size, strings.Join(methods, ", ")) diff --git a/cmd/pull/pull_test.go b/cmd/pull/pull_test.go index dd77e67b..fb45fb11 100644 --- a/cmd/pull/pull_test.go +++ b/cmd/pull/pull_test.go @@ -250,7 +250,7 @@ func TestCheckoutDownloadedFilesRejectsInvalidCachedObject(t *testing.T) { files := []pointerFile{{Name: "data/file.bin", Oid: oid, Size: 100}} progress.OnPlan(toPullFiles(files)) - err = checkoutDownloadedFiles(files, progress) + err = checkoutDownloadedFiles(files, progress, false) if err == nil { t.Fatal("expected checkoutDownloadedFiles to reject invalid cached object") } @@ -293,7 +293,7 @@ func TestCheckedOutContentCleansBackToPointer(t *testing.T) { progress := internaltransfer.NewPullProgressRenderer(io.Discard) files := []pointerFile{{Name: "data/file.bin", Oid: oid, Size: int64(len(payload))}} progress.OnPlan(toPullFiles(files)) - if err := checkoutDownloadedFiles(files, progress); err != nil { + if err := checkoutDownloadedFiles(files, progress, false); err != nil { t.Fatalf("checkoutDownloadedFiles: %v", err) } @@ -316,6 +316,51 @@ func TestCheckedOutContentCleansBackToPointer(t *testing.T) { } } +func TestCheckoutDownloadedFilesFromReadOnlyRemoteSetsReadOnlyPermission(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX permission bits are not supported on Windows") + } + + repo := t.TempDir() + oldWD, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + if err := os.Chdir(repo); err != nil { + t.Fatalf("chdir repo: %v", err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + + payload := []byte("read-only remote payload") + sum := sha256.Sum256(payload) + oid := hex.EncodeToString(sum[:]) + cachePath, err := lfs.ObjectPath(gitrepo.LFSObjectsPath, oid) + if err != nil { + t.Fatalf("ObjectPath: %v", err) + } + if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { + t.Fatalf("mkdir cache dir: %v", err) + } + if err := os.WriteFile(cachePath, payload, 0o644); err != nil { + t.Fatalf("write cache: %v", err) + } + + files := []pointerFile{{Name: "data/file.bin", Oid: oid, Size: int64(len(payload))}} + progress := internaltransfer.NewPullProgressRenderer(io.Discard) + progress.OnPlan(toPullFiles(files)) + if err := checkoutDownloadedFiles(files, progress, true); err != nil { + t.Fatalf("checkoutDownloadedFiles: %v", err) + } + + info, err := os.Stat(files[0].Name) + if err != nil { + t.Fatalf("stat checked-out file: %v", err) + } + if got, want := info.Mode().Perm(), os.FileMode(0o444); got != want { + t.Fatalf("checked-out permissions = %o, want %o", got, want) + } +} + func TestRefreshGitIndexForHydratedFilesClearsDirtyStatus(t *testing.T) { repo := t.TempDir() gitDRS := buildGitDRSBinaryForTest(t) @@ -335,9 +380,13 @@ func TestRefreshGitIndexForHydratedFilesClearsDirtyStatus(t *testing.T) { worktreePath := filepath.Join(repo, "sample.bin") payload := []byte("hello world payload") - sum := sha256.Sum256(payload) - oid := hex.EncodeToString(sum[:]) - writePointerFile(t, worktreePath, oid, strconv.Itoa(len(payload))) + oid := "drs://drs.anv0:v2_example-without-sha256" + pointer := "version https://calypr.github.io/spec/v1\n" + + "oid " + oid + "\n" + + "size " + strconv.Itoa(len(payload)) + "\n" + if err := os.WriteFile(worktreePath, []byte(pointer), 0o644); err != nil { + t.Fatalf("write DRS pointer: %v", err) + } runGitCmdTest(t, repo, "add", ".gitattributes", "sample.bin") runGitCmdTest(t, repo, "commit", "-m", "commit pointer") diff --git a/cmd/push/main.go b/cmd/push/main.go index a1758d09..9b52f60e 100644 --- a/cmd/push/main.go +++ b/cmd/push/main.go @@ -58,12 +58,20 @@ var Cmd = &cobra.Command{ return err } } - drsClient, err := remoteruntime.New(cfg, remote, myLogger) if err != nil { myLogger.DebugContext(ctx, "create remote client failed", "error", err) return err } + if drsClient.IsReadOnly() || !drsClient.CanUpload() || !drsClient.CanRegister() { + return fmt.Errorf( + "remote %q is read-only: git drs push cannot upload files to Terra or register DRS objects\n"+ + "no files were uploaded, and you do not need to back out a commit that references existing Terra data\n"+ + "to publish the commit and its DRS references, use ordinary git push to a Git remote; this pushes only Git metadata and does not upload files to Terra", + remote, + ) + } + drsClient.ForceUpload = pushForceUpload state, err := resolveSyncRefState(ctx, string(remote)) if err != nil { diff --git a/cmd/push/push_test.go b/cmd/push/push_test.go index 69f16c4e..b8a79962 100644 --- a/cmd/push/push_test.go +++ b/cmd/push/push_test.go @@ -6,6 +6,7 @@ import ( "github.com/calypr/git-drs/internal/config" "github.com/calypr/git-drs/internal/testutils" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestPushCmdArgs(t *testing.T) { @@ -38,3 +39,25 @@ func TestPushRun_DefaultRemoteError(t *testing.T) { err := Cmd.RunE(Cmd, []string{}) assert.Error(t, err) } + +func TestPushRun_ReadOnlyTerraRemoteExplainsWhatWasNotPushed(t *testing.T) { + tmpDir := testutils.SetupTestGitRepo(t) + testutils.CreateTestConfig(t, tmpDir, &config.Config{ + DefaultRemote: "anvil", + Remotes: map[config.Remote]config.RemoteSelect{ + "anvil": {Terra: &config.TerraRemote{ + Endpoint: "https://data.terra.bio", + Mode: "read-only", + }}, + }, + }) + + err := Cmd.RunE(Cmd, nil) + require.Error(t, err) + assert.ErrorContains(t, err, `remote "anvil" is read-only`) + assert.ErrorContains(t, err, "cannot upload files to Terra") + assert.ErrorContains(t, err, "no files were uploaded") + assert.ErrorContains(t, err, "do not need to back out a commit") + assert.ErrorContains(t, err, "ordinary git push to a Git remote") + assert.ErrorContains(t, err, "pushes only Git metadata") +} diff --git a/cmd/remote/add/init.go b/cmd/remote/add/init.go index 4248b546..616a6328 100644 --- a/cmd/remote/add/init.go +++ b/cmd/remote/add/init.go @@ -9,24 +9,46 @@ var ( localPassword string localUsername string noSkipSmudge bool + terraEndpoint string + terraAuth string + terraMode string ) // Cmd line declaration var Cmd = &cobra.Command{ Use: "add", - Short: "add server access for git-drs", + Short: "Add a DRS server using an endpoint or built-in preset", + Args: cobra.RangeArgs(1, 2), + RunE: runUnified, } func init() { + Cmd.PersistentFlags().StringVar(&scopeFlag, "scope", "", "Provider-specific organization/project scope") + Cmd.PersistentFlags().StringVar(&authFlag, "auth", "auto", "Authentication method") + Cmd.PersistentFlags().StringVar(&credentialFlag, "credential", "", "Credential source (never an inline secret)") + Cmd.PersistentFlags().StringVar(&providerFlag, "provider", "auto", "DRS provider adapter") + Cmd.PersistentFlags().StringVar(&storageFlag, "storage", "", "Advanced publishing bucket/prefix") + Cmd.PersistentFlags().StringVar(&checkoutFlag, "checkout", "", "Checkout mode: pointers or hydrate") Gen3Cmd.Flags().StringVar(&credFile, "cred", "", "[gen3] Import a Gen3 credential file into this profile") Gen3Cmd.Flags().StringVar(&fenceToken, "token", "", "[gen3] Use a temporary bearer token issued from fence") Gen3Cmd.Flags().StringVar(&selectedBucket, "bucket", "", "[gen3] Select a specific visible bucket when multiple buckets match the scope") Gen3Cmd.Flags().BoolVar(&noSkipSmudge, "no-skip-smudge", false, "Disable skipping smudge filter (force downloading file contents during checkout)") Cmd.AddCommand(Gen3Cmd) + Gen3Cmd.Deprecated = "use 'git drs remote add --provider gen3 --scope --credential '" + Gen3Cmd.Hidden = true LocalCmd.Flags().StringVar(&selectedBucket, "bucket", "", "Select a specific visible bucket when multiple buckets match the scope") LocalCmd.Flags().StringVar(&localUsername, "username", "", "Username for local DRS HTTP basic auth") LocalCmd.Flags().StringVar(&localPassword, "password", "", "Password for local DRS HTTP basic auth") LocalCmd.Flags().BoolVar(&noSkipSmudge, "no-skip-smudge", false, "Disable skipping smudge filter (force downloading file contents during checkout)") Cmd.AddCommand(LocalCmd) + LocalCmd.Deprecated = "use the unified endpoint form" + LocalCmd.Hidden = true + + TerraCmd.Flags().StringVar(&terraEndpoint, "drs-endpoint", "", "Terra DRS service base URL") + TerraCmd.Flags().StringVar(&terraAuth, "auth", "", "Terra authentication method (google-adc)") + TerraCmd.Flags().StringVar(&terraMode, "mode", "", "Terra remote mode (read-only)") + Cmd.AddCommand(TerraCmd) + TerraCmd.Deprecated = "use 'git drs remote add terra'" + TerraCmd.Hidden = true } diff --git a/cmd/remote/add/terra.go b/cmd/remote/add/terra.go new file mode 100644 index 00000000..07c53f4b --- /dev/null +++ b/cmd/remote/add/terra.go @@ -0,0 +1,61 @@ +package add + +import ( + "fmt" + "net/url" + "strings" + + "github.com/calypr/git-drs/cmd/initialize" + "github.com/calypr/git-drs/internal/config" + "github.com/calypr/git-drs/internal/drslog" + "github.com/spf13/cobra" +) + +var TerraCmd = &cobra.Command{ + Use: "terra ", + Short: "Add a read-only Terra DRS remote", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return runUnified(cmd, []string{"terra"}) + } + remoteName := strings.TrimSpace(args[0]) + if remoteName == "" { + return fmt.Errorf("remote name is required") + } + endpoint, err := validateTerraOptions(terraEndpoint, terraAuth, terraMode) + if err != nil { + return err + } + if err := initialize.EnsureInitialized(drslog.GetLogger()); err != nil { + return fmt.Errorf("failed to initialize repository: %w", err) + } + + newConfig, err := config.UpdateRemote(config.Remote(remoteName), config.RemoteSelect{ + Terra: &config.TerraRemote{Endpoint: endpoint, Auth: terraAuth, Mode: terraMode}, + }) + if err != nil { + return fmt.Errorf("failed to update remote config: %w", err) + } + fmt.Printf("Added remote '%s'. Config: %v\n", remoteName, newConfig.GetRemote(config.Remote(remoteName))) + return nil + }, +} + +func validateTerraOptions(endpoint, auth, mode string) (string, error) { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return "", fmt.Errorf("--drs-endpoint is required") + } + u, err := url.ParseRequestURI(endpoint) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil { + return "", fmt.Errorf("--drs-endpoint must be an HTTPS URL without credentials") + } + if auth != "google-adc" { + return "", fmt.Errorf("--auth must be google-adc") + } + if mode != "read-only" { + return "", fmt.Errorf("--mode must be read-only") + } + return u.String(), nil +} diff --git a/cmd/remote/add/terra_test.go b/cmd/remote/add/terra_test.go new file mode 100644 index 00000000..c203a6e8 --- /dev/null +++ b/cmd/remote/add/terra_test.go @@ -0,0 +1,58 @@ +package add + +import ( + "testing" + + "github.com/calypr/git-drs/internal/config" + "github.com/calypr/git-drs/internal/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAddTerraRemoteCommand(t *testing.T) { + assert.Equal(t, "terra ", TerraCmd.Use) + assert.NotNil(t, TerraCmd.Flag("drs-endpoint")) + assert.NotNil(t, TerraCmd.Flag("auth")) + assert.NotNil(t, TerraCmd.Flag("mode")) +} + +func TestValidateTerraOptions(t *testing.T) { + endpoint, err := validateTerraOptions("https://data.terra.bio", "google-adc", "read-only") + require.NoError(t, err) + assert.Equal(t, "https://data.terra.bio", endpoint) + + for name, options := range map[string][3]string{ + "missing endpoint": {"", "google-adc", "read-only"}, + "HTTP endpoint": {"http://data.terra.bio", "google-adc", "read-only"}, + "URL credentials": {"https://user:pass@data.terra.bio", "google-adc", "read-only"}, + "unsupported auth": {"https://data.terra.bio", "token", "read-only"}, + "writable mode": {"https://data.terra.bio", "google-adc", "write"}, + } { + t.Run(name, func(t *testing.T) { + _, err := validateTerraOptions(options[0], options[1], options[2]) + assert.Error(t, err) + }) + } +} + +func TestTerraRemoteAddPersistsConfig(t *testing.T) { + testutils.SetupTestGitRepo(t) + terraEndpoint = "https://data.terra.bio" + terraAuth = "google-adc" + terraMode = "read-only" + t.Cleanup(func() { + terraEndpoint = "" + terraAuth = "" + terraMode = "" + }) + + require.NoError(t, TerraCmd.RunE(TerraCmd, []string{"anvil"})) + cfg, err := config.LoadConfig() + require.NoError(t, err) + remote := cfg.Remotes[config.Remote("anvil")].Terra + require.NotNil(t, remote) + assert.Equal(t, "https://data.terra.bio", remote.Endpoint) + assert.Equal(t, "google-adc", remote.Auth) + assert.Equal(t, "read-only", remote.Mode) + assert.Equal(t, config.Remote("anvil"), cfg.DefaultRemote) +} diff --git a/cmd/remote/add/unified.go b/cmd/remote/add/unified.go new file mode 100644 index 00000000..0831b34c --- /dev/null +++ b/cmd/remote/add/unified.go @@ -0,0 +1,156 @@ +package add + +import ( + "fmt" + "net/url" + "regexp" + "strings" + + "github.com/calypr/git-drs/cmd/initialize" + "github.com/calypr/git-drs/internal/config" + "github.com/calypr/git-drs/internal/drslog" + "github.com/calypr/git-drs/internal/gitrepo" + "github.com/calypr/git-drs/internal/presets" + "github.com/spf13/cobra" +) + +var ( + scopeFlag, authFlag, credentialFlag, providerFlag string + storageFlag, checkoutFlag string +) + +func runUnified(cmd *cobra.Command, args []string) error { + if len(args) < 1 || len(args) > 2 { + return fmt.Errorf("expected or ") + } + name, selector := "", args[0] + if len(args) == 2 { + name, selector = strings.TrimSpace(args[0]), args[1] + } + if strings.HasPrefix(selector, "registry:") { + return fmt.Errorf("registry selector %q requires registry discovery, which is unavailable; pass the service HTTPS URL explicitly", selector) + } + + preset, isPreset, err := presets.Lookup(selector) + if err != nil { + return err + } + endpoint, provider, auth, presetName, registryID, version := selector, "auto", "auto", "", "", 0 + if isPreset { + endpoint, provider, auth = preset.Endpoint, preset.Provider, preset.Auth + presetName, registryID, version = preset.Alias, preset.RegistryServiceID, presets.CatalogVersion + } + if providerFlag != "" && providerFlag != "auto" { + provider = providerFlag + } + if authFlag != "" && authFlag != "auto" { + auth = authFlag + } + if !validChoice(provider, "auto", "ga4gh", "gen3", "terra", "cgc", "synapse") { + return fmt.Errorf("unsupported provider %q", provider) + } + if provider == "auto" { + return fmt.Errorf("cannot determine provider for endpoint %q; specify --provider", endpoint) + } + if !validAuth(auth) { + return fmt.Errorf("unsupported authentication method %q", auth) + } + if err := validateProviderAuth(provider, auth); err != nil { + return err + } + u, err := url.ParseRequestURI(strings.TrimSpace(endpoint)) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil { + return fmt.Errorf("endpoint must be an HTTPS URL without credentials, or a built-in alias") + } + if name == "" { + name = deriveRemoteName(presetName, u.Hostname()) + } + if name == "" { + return fmt.Errorf("could not derive a remote name; use the explicit-name form") + } + if !regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`).MatchString(name) { + return fmt.Errorf("invalid remote name %q", name) + } + if scopeFlag != "" { + if _, _, err := parseScopeArg(scopeFlag); err != nil { + return err + } + } + if credentialFlag != "" && !validCredentialSource(credentialFlag) { + return fmt.Errorf("invalid credential source %q; use env:, file:, helper:, profile:, or stdin", credentialFlag) + } + if checkoutFlag != "" && !validChoice(checkoutFlag, "pointers", "hydrate") { + return fmt.Errorf("--checkout must be pointers or hydrate") + } + + if err := initialize.EnsureInitialized(drslog.GetLogger()); err != nil { + return fmt.Errorf("failed to initialize repository: %w", err) + } + if cfg, loadErr := config.LoadConfig(); loadErr == nil { + if _, exists := cfg.Remotes[config.Remote(name)]; exists { + return fmt.Errorf("remote %q already exists; choose an explicit name or remove it first", name) + } + } + r := &config.GenericRemote{Endpoint: u.String(), Provider: provider, Auth: auth, Credential: credentialFlag, Scope: scopeFlag, Storage: storageFlag, Checkout: checkoutFlag, Preset: presetName, PresetVersion: version, RegistryServiceID: registryID} + fmt.Fprintf(cmd.OutOrStdout(), "Remote: %s\nEndpoint: %s\nProvider: %s\nAuth: %s\n", name, r.Endpoint, r.Provider, r.Auth) + if r.Preset != "" { + fmt.Fprintf(cmd.OutOrStdout(), "Preset: %s (built-in catalog v%d)\n", r.Preset, r.PresetVersion) + } + if _, err := config.UpdateRemote(config.Remote(name), config.RemoteSelect{Generic: r}); err != nil { + return fmt.Errorf("save remote: %w", err) + } + if err := configureRepoRemote(name, r.Endpoint); err != nil { + return err + } + if checkoutFlag != "" { + skip := "true" + if checkoutFlag == "hydrate" { + skip = "false" + } + if err := gitrepo.SetGitConfigOptions(map[string]string{"drs.skipsmudge": skip}); err != nil { + return err + } + } + fmt.Fprintln(cmd.OutOrStdout(), "Remote saved.") + return nil +} + +func deriveRemoteName(alias, host string) string { + if alias != "" { + return alias + } + host = strings.ToLower(strings.TrimSpace(host)) + host = strings.TrimPrefix(host, "www.") + return regexp.MustCompile(`[^a-z0-9._-]+`).ReplaceAllString(host, "-") +} +func validChoice(v string, choices ...string) bool { + for _, c := range choices { + if v == c { + return true + } + } + return false +} +func validAuth(v string) bool { + return validChoice(v, "auto", "none", "bearer", "basic", "google-adc", "provider-helper") || strings.HasPrefix(v, "provider-helper:") +} +func validateProviderAuth(provider, auth string) error { + if provider != "gen3" { + return nil + } + if validChoice(auth, "bearer", "provider-helper", "provider-helper:gen3-profile") { + return nil + } + return fmt.Errorf("authentication method %q is not supported by the Gen3 remote adapter; use bearer or provider-helper:gen3-profile", auth) +} +func validCredentialSource(v string) bool { + if v == "stdin" { + return true + } + for _, p := range []string{"env:", "file:", "helper:", "profile:"} { + if strings.HasPrefix(v, p) && len(v) > len(p) { + return true + } + } + return false +} diff --git a/cmd/remote/add/unified_test.go b/cmd/remote/add/unified_test.go new file mode 100644 index 00000000..42086c66 --- /dev/null +++ b/cmd/remote/add/unified_test.go @@ -0,0 +1,98 @@ +package add + +import ( + "bytes" + "os" + "testing" + + "github.com/calypr/git-drs/internal/config" + "github.com/calypr/git-drs/internal/testutils" +) + +func TestUnifiedAddExpandsPreset(t *testing.T) { + testutils.SetupTestGitRepo(t) + resetUnifiedFlags(t) + credentialFlag = "env:CGC_TOKEN" + var out bytes.Buffer + Cmd.SetOut(&out) + if err := runUnified(Cmd, []string{"cgc"}); err != nil { + t.Fatal(err) + } + cfg, err := config.LoadConfig() + if err != nil { + t.Fatal(err) + } + r := cfg.Remotes["cgc"].Generic + if r == nil || r.Provider != "cgc" || r.Auth != "bearer" || r.PresetVersion != 1 || r.Credential != "env:CGC_TOKEN" { + t.Fatalf("unexpected remote: %+v", r) + } + if out.String() == "" { + t.Fatal("expected effective configuration output") + } +} + +func TestUnifiedAddDerivesURLNameAndRejectsHTTP(t *testing.T) { + testutils.SetupTestGitRepo(t) + resetUnifiedFlags(t) + if err := runUnified(Cmd, []string{"http://drs.example.org"}); err == nil { + t.Fatal("expected HTTP rejection") + } + providerFlag = "ga4gh" + if err := runUnified(Cmd, []string{"https://drs.example.org"}); err != nil { + t.Fatal(err) + } + cfg, err := config.LoadConfig() + if err != nil { + t.Fatal(err) + } + if cfg.Remotes["drs.example.org"].Generic == nil { + t.Fatal("derived remote not persisted") + } +} + +func TestUnifiedAddRejectsURLWithoutProviderBeforeInitializing(t *testing.T) { + repo := testutils.SetupTestGitRepo(t) + resetUnifiedFlags(t) + + err := runUnified(Cmd, []string{"https://drs.example.org"}) + if err == nil { + t.Fatal("expected provider auto rejection") + } + if _, statErr := os.Stat(repo + "/.git-drs"); !os.IsNotExist(statErr) { + t.Fatalf("remote validation modified repository state: .git-drs stat error = %v", statErr) + } + cfg, loadErr := config.LoadConfig() + if loadErr != nil { + t.Fatal(loadErr) + } + if len(cfg.Remotes) != 0 { + t.Fatalf("remote validation persisted configuration: %+v", cfg.Remotes) + } +} + +func TestUnifiedAddRejectsUnsupportedGen3AuthenticationBeforeInitializing(t *testing.T) { + for _, auth := range []string{"auto", "none", "basic", "google-adc", "provider-helper:other"} { + t.Run(auth, func(t *testing.T) { + repo := testutils.SetupTestGitRepo(t) + resetUnifiedFlags(t) + providerFlag = "gen3" + authFlag = auth + + err := runUnified(Cmd, []string{"https://gen3.example"}) + if err == nil { + t.Fatalf("expected Gen3 authentication method %q to be rejected", auth) + } + if _, statErr := os.Stat(repo + "/.git-drs"); !os.IsNotExist(statErr) { + t.Fatalf("remote validation modified repository state: .git-drs stat error = %v", statErr) + } + }) + } +} + +func resetUnifiedFlags(t *testing.T) { + old := []string{scopeFlag, authFlag, credentialFlag, providerFlag, storageFlag, checkoutFlag} + scopeFlag, authFlag, credentialFlag, providerFlag, storageFlag, checkoutFlag = "", "auto", "", "auto", "", "" + t.Cleanup(func() { + scopeFlag, authFlag, credentialFlag, providerFlag, storageFlag, checkoutFlag = old[0], old[1], old[2], old[3], old[4], old[5] + }) +} diff --git a/cmd/remote/list.go b/cmd/remote/list.go index a7a3035d..b7e2911e 100644 --- a/cmd/remote/list.go +++ b/cmd/remote/list.go @@ -54,6 +54,12 @@ var ListCmd = &cobra.Command{ } else if remoteSelect.Local != nil { remoteType = string(config.LocalServerType) remote = remoteSelect.Local + } else if remoteSelect.Terra != nil { + remoteType = string(config.TerraServerType) + remote = remoteSelect.Terra + } else if remoteSelect.Generic != nil { + remoteType = remoteSelect.Generic.Provider + remote = remoteSelect.Generic } else { remoteType = "unknown" } diff --git a/cmd/root.go b/cmd/root.go index 320592f8..bd78ef18 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -14,6 +14,7 @@ import ( "github.com/calypr/git-drs/cmd/lsfiles" "github.com/calypr/git-drs/cmd/ping" "github.com/calypr/git-drs/cmd/precommit" + "github.com/calypr/git-drs/cmd/preset" "github.com/calypr/git-drs/cmd/pull" "github.com/calypr/git-drs/cmd/push" "github.com/calypr/git-drs/cmd/query" @@ -50,6 +51,7 @@ func init() { RootCmd.AddCommand(pull.Cmd) RootCmd.AddCommand(push.Cmd) RootCmd.AddCommand(precommit.Cmd) + RootCmd.AddCommand(preset.Cmd) RootCmd.AddCommand(addref.Cmd) RootCmd.AddCommand(addurl.Cmd) RootCmd.AddCommand(deleteCmd.Cmd) diff --git a/cmd/smudge/main.go b/cmd/smudge/main.go index faebd995..964df49c 100644 --- a/cmd/smudge/main.go +++ b/cmd/smudge/main.go @@ -10,6 +10,7 @@ import ( "github.com/calypr/git-drs/internal/drslog" internalfilter "github.com/calypr/git-drs/internal/filter" "github.com/calypr/git-drs/internal/remoteruntime" + "github.com/calypr/git-drs/internal/resolver" internaltransfer "github.com/calypr/git-drs/internal/transfer" "github.com/spf13/cobra" ) @@ -64,7 +65,20 @@ func runSmudge(cmd *cobra.Command, args []string) error { var downloadFn internalfilter.SmudgeDownloadFunc if !internalfilter.ShouldSkipSmudge() { + var terraResolver resolver.Resolver + if drsCtx.RemoteType == config.TerraServerType { + terraResolver, err = resolver.NewAnVIL(ctx, drsCtx.Endpoint) + if err != nil { + return fmt.Errorf("smudge: create Terra resolver: %w", err) + } + } downloadFn = func(callCtx context.Context, oid, cachePath string) error { + if terraResolver != nil { + if len(oid) >= 2 && oid[:2] == "//" { + oid = "drs:" + oid + } + return resolver.DownloadToCache(callCtx, terraResolver, oid, cachePath) + } return internaltransfer.DownloadToCachePath(callCtx, drsCtx, oid, cachePath) } } diff --git a/docs/adding-s3-files.md b/docs/adding-s3-files.md index 0151d57a..0bf1d8d2 100644 --- a/docs/adding-s3-files.md +++ b/docs/adding-s3-files.md @@ -74,9 +74,10 @@ If SHA256 is unknown, omit `--sha256`. Behavior: 1. `add-url` performs object metadata lookup (HEAD/attributes). -2. A deterministic placeholder OID is derived from remote object metadata. -3. A pointer file and local DRS metadata are written. -4. `git drs push` performs metadata registration. +2. A deterministic placeholder/local OID is derived from remote object metadata. +3. A pointer file and local DRS metadata are written; the placeholder is not recorded as a content checksum. +4. The provider URL/source metadata remains the retrieval identity until content is downloaded. +5. `git drs push` performs metadata registration. ```bash git drs track "data/*.bin" diff --git a/docs/anvil-data-explorer.png b/docs/anvil-data-explorer.png new file mode 100644 index 00000000..65adf8ae Binary files /dev/null and b/docs/anvil-data-explorer.png differ diff --git a/docs/anvil-terra-poc-presentation.md b/docs/anvil-terra-poc-presentation.md new file mode 100644 index 00000000..80e5e6ab --- /dev/null +++ b/docs/anvil-terra-poc-presentation.md @@ -0,0 +1,329 @@ +--- +marp: true +title: AnVIL + Terra x git-drs — reference POC +description: A reference proof of concept for version-controlled dataset layouts +paginate: true +theme: git-drs +--- + + + + + + +# AnVIL + Terra meets **git-drs** + +Version-controlled dataset layouts—without copying controlled-access payloads into Git. + +Reference proof of concept · July 2026 + +--- + +## Git moves references. AnVIL moves bytes. + +| User A | Git | User B | AnVIL | +|---|---|---|---| +| Add references with ADC-authorized metadata validation | Commit canonical DRS URI, size, and checksum | Clone, configure the Terra remote, and authenticate independently | Hydrate from a fresh authorized URL and verify bytes | + +
+

0 payload bytes

committed to Git

+

2 identities

authenticate independently

+

1 reference

stays canonical and portable

+
+ +> **Security boundary:** Git visibility reveals paths and DRS identifiers; it never grants authorization to the underlying data. + +--- + +## One reference model, four perspectives + +
+

Data reference author

Publish a reviewable, versioned dataset layout—one object or a manifest—without downloading payloads.

+

Data consumer

Clone pointers, authenticate independently, then hydrate everything or only paths matching a pattern.

+

Unauthorized reader

Inspect repository history but receive a clear denial when attempting controlled-data hydration.

+

Repository maintainer

Review deterministic pointers, diagnose provider failures, and run clean-clone acceptance tests.

+
+ +**Out of scope:** uploads, record mutation, provider copying, remote GC, and workspace-table sync. + +--- + +## Add once. Publish with ordinary Git. + +
+
+

+gcloud auth application-default login
+
+git drs add-ref --remote anvil \
+  drs://authority/object-1 data/sample.cram
+
+# Validate a batch before writing
+git drs add-ref --remote anvil \
+  --manifest references.tsv --dry-run
+
+git add .gitattributes data/
+git commit -m "Add AnVIL data references"
+git push
+
+
+
+

Published

+
    +
  • Canonical DRS URI pointers
    • +
  • Paths and Git history
    • +
  • .gitattributes
    • +
  • Only portable pointer-management state
    • +
+

Never published

+
    +
  • Payload bytes or cache content
    • +
  • Tokens, headers, or ADC files
    • +
  • Signed download URLs
    • +
+
+
+ +--- + +## A clean clone plus explicit remote setup + +
+
+

+gcloud auth application-default login
+
+git clone <git-repository>
+cd <repository>
+
+# .git/config is not cloned; recreate public remote settings.
+git drs remote add anvil terra --checkout hydrate
+
+# Hydrate every authorized reference
+git drs pull
+
+# Or only one dataset slice
+git drs pull -I "data/*.cram"
+
+

No author cache, local Git config, token, or signed URL is transferred.

+
+
+ + + + + + + + + +
StageAction
GitClone pointer and .gitattributes
User BConfigure remote; supply ADC; choose paths
ResolverFetch current metadata and fresh access
CacheDownload, verify, atomically promote
WorktreeHydrate only after validation
+
+
+ +--- + + + +## AnVIL manifest to pointer-only GitHub repo + +
+
+

1. Configure and add references

+

Select the dataset in the AnVIL Data Explorer, then download its manifest.

+

+git init
+git drs remote add anvil terra
+
+# Download the TSV from AnVIL Data Explorer first.
+manifest=/tmp/anvil-manifest-38dc7537.tsv
+scripts/anvil-add-ref-commands.sh "$manifest" \
+  > /tmp/add-anvil-refs.sh
+cat /tmp/add-anvil-refs.sh
+bash /tmp/add-anvil-refs.sh
+
+
+
+

2. Commit, hydrate, and publish

+

+git add .gitattributes '*.tsv'
+git commit -m "Add references to AnVIL data"
+
+# Materialize only TSVs locally.
+git drs pull -I "*.tsv"
+git status
+
+git branch -M main
+git remote add origin \
+  https://github.com/bwalsh/\
+ANVIL_1000G_PRIMED_data_model.git
+git push -u origin main
+
+
+
+ +> **Result:** the worktree contains hydrated TSV data; GitHub contains only DRS pointers and `.gitattributes`. Every clone must configure its own Terra remote. + +--- + +## Portable pointer + safe configuration + +
+
+

Tracked pointer

+

+version https://calypr.github.io/spec/v1
+oid drs://authority/object-1
+size 987654321
+sha256 8d969eef…
+
+

The DRS URI stays canonical. A checksum describes content; it does not identify the AnVIL record.

+
+
+

Repository-local Git configuration

+

+git drs remote add anvil terra --checkout hydrate
+
+

Remote metadata lives only in clone-local .git/config; every clone must recreate it. Credentials remain in the ADC provider store.

+
+
+ +--- + +## Provider-neutral resolution breaks Syfon coupling + +```text +Commands Resolver interface Identities +add-ref · pull · ping -> GetObject · GetAccess -> remote · cache · content + | + +----------+----------+ + | | + AnVILResolver SyfonResolver + Google ADC · routing Gen3 · local behavior +``` + +### Design rules + +- Commands depend on one resolver contract. +- The configured remote selects the resolver. +- Authentication and provider routing stay behind the interface. +- There is no user-facing `--remote-type` switch. + +--- + +## Three identities—not one overloaded OID + +
+
REMOTE

Canonical DRS URI

Normalized provider record identity. Committed to Git and used for object and access resolution.

+
LOCAL

Cache OID

Filesystem-safe SHA256 of a versioned prefix plus normalized DRS URI. Never replaces remote identity.

+
CONTENT

Content SHA256

Durable integrity metadata, when supplied. Used to verify bytes—not to locate the record.

+
+ +```text +cache_oid = sha256("git-drs-anvil-ref:v1\n" + normalized_drs_uri) +``` + +> **Download invariant:** fresh access → temporary file → verify size/checksum → atomic cache promotion → hydrate worktree. + +--- + + + +## Component status: implemented, partial, and missing + +
+
IMPLEMENTED

Focused coverage

  • ADC-backed resolver contract
  • Canonical pointer and cache key
  • Atomic size/SHA256 validation
  • Terra ping and push refusal
+
PARTIAL

Workflow coverage

  • Manifest validation is sequential
  • Selective pull is covered in isolation
  • Cache reuse is per clone
  • Remote setup is clone-local
+
MISSING

POC acceptance

  • Independent User A/User B journey
  • Production AnVIL contract proof
  • Expired-URL retry and concurrency
  • Denied-user and leak audit
+
+ +| Arrange | Act | Assert | +|---|---|---| +| Separate homes, config, ADC, caches | Commit → clone → authenticate → pull | Verified bytes; isolated credentials | + +--- + + + +## Most original blockers are now closed + +
+
+

Implemented

+
    +
  • ADC-backed AnVILResolver handles metadata and access.
    • +
  • Terra add-ref and pull use provider-neutral resolution.
    • +
  • Remote config selects behavior; --remote-type is deprecated.
    • +
  • Terra pointers retain DRS URI, size, and optional SHA256.
    • +
  • Cache keys are separate from content checksums.
    • +
+
+
+

Still incomplete

+
    +
  • Every fresh clone must manually recreate the Terra remote in .git/config.
    • +
  • The production endpoint, OAuth scope, and object/access contracts are not certified end to end.
    • +
  • Manifest resolution and downloads lack bounded concurrency and retry.
    • +
  • An expired download URL is not re-resolved after an HTTP failure.
    • +
  • Independent two-user, denied-user, and credential-leak acceptance remain unverified.
    • +
+
+
+ +> **Bottom line:** an implemented vertical slice still needs production-contract validation, resilience, and end-to-end proof. + +--- + + + +## Turn the vertical slice into a proven POC + +
+
P0 · VERIFY

Prove production fit

  • Confirm endpoint and OAuth scopes
  • Certify object/access contracts
  • Test slash and compact DRS IDs
  • Run clean two-user clone/pull
  • Audit logs and history
+
P1 · HARDEN

Make it resilient

  • Safe clone setup mechanism
  • Expired-URL re-resolution
  • Bounded retry and concurrency
  • Cancellation and diagnostics
  • Clean-environment CI
+
P2 · GENERALIZE

Keep DRS composable

  • Provider-neutral resolver contract
  • CGC and Synapse fixtures
  • Capability discovery
  • Keep publishing provider-specific
  • Defer mutation and copying
+
+ +--- + +## Decisions needed to start the POC + +1. Which authoritative AnVIL resolver contract, endpoint, and OAuth scopes will we certify? +2. What stable test objects cover checksummed, non-checksummed, large, and denied cases? +3. Is the DRS-URI pointer extension compatible with every parser that must read it? +4. Should consumers always run `remote add`, or may a tracked, non-secret bootstrap configure `.git/config`? +5. Who owns the independent-user acceptance environment and compatibility matrix? + +### Questions? + +Detailed design and acceptance criteria are in [`docs/anvil-terra-poc.md`](anvil-terra-poc.md). diff --git a/docs/anvil-terra-poc.md b/docs/anvil-terra-poc.md new file mode 100644 index 00000000..d0e83523 --- /dev/null +++ b/docs/anvil-terra-poc.md @@ -0,0 +1,569 @@ +# AnVIL/Terra DRS Reference Proof of Concept + +## Purpose + +This document defines a focused proof of concept (POC) for using `git-drs` with the AnVIL/Terra DRS ecosystem. The POC is a **reference-only workflow**: data already exists behind AnVIL DRS records, and Git stores portable references to that data rather than uploading or copying payload bytes. + +The intended end-to-end flow is: + +1. An authorized user adds several AnVIL DRS references to a Git repository. +2. The user commits the generated pointer files and pushes the Git repository with ordinary `git push`. +3. A second authorized user clones the repository. +4. The second user authenticates with their own Terra/AnVIL identity and runs `git drs pull` to hydrate the referenced files. + +## Scope + +The POC includes: + +- a read-only Terra remote; +- Google Application Default Credentials (ADC) authentication; +- resolution of AnVIL DRS URIs to authorized access URLs; +- single-reference and manifest-driven batch `add-ref` workflows; +- portable pointer files containing canonical DRS URIs; +- non-secret, repository-level remote configuration; +- selective and complete hydration after a fresh clone; +- size and checksum verification where metadata is available. + +The POC does not include: + +- uploading payload bytes to AnVIL; +- creating, editing, or deleting AnVIL DRS records; +- copying AnVIL records into Syfon or Gen3; +- destructive remote garbage collection; +- automatic Terra workspace-table synchronization; +- support for every DRS provider or Google authentication mode. + +## User Stories + +### Data reference author + +As an authorized AnVIL user, I want to add one or more existing DRS objects to a Git repository so that I can publish a reviewable, versioned dataset layout without downloading or committing the payload bytes. + +Acceptance criteria: + +- I can add one reference by DRS URI and destination path. +- I can add many references from a manifest. +- The command validates every DRS URI using my identity. +- Generated pointers contain no payload data, access token, signed URL, or other secret. +- The references can be committed and published with ordinary Git commands. + +### Data consumer + +As a second authorized AnVIL user, I want to clone the Git repository and hydrate its referenced files using my own identity so that I can reproduce the repository's data layout without receiving the author's credentials or clone-local state. + +Acceptance criteria: + +- A fresh clone contains portable pointer files. +- The repository supplies non-secret Terra remote configuration. +- I authenticate independently using Google ADC. +- `git drs pull` hydrates all authorized references. +- `git drs pull -I ` hydrates only selected references. +- Downloaded content is checked against its expected size and checksum when available. + +### Unauthorized repository reader + +As a person who can read the Git repository but lacks AnVIL data authorization, I may inspect paths and DRS references, but I must not be able to download controlled-access payloads. + +Acceptance criteria: + +- Cloning Git does not grant data access. +- Pull reports a clear authorization error. +- No committed file or log exposes another user's credential or signed download URL. + +### Repository maintainer + +As a repository maintainer, I want deterministic pointers, actionable diagnostics, and repeatable tests so that reference changes are reviewable and failures can be distinguished from authorization problems. + +## User Experience + +### Prerequisites + +Both users need: + +- Git and `git-drs`; +- access to the Git repository; +- a Google identity authorized for the referenced AnVIL data; +- working Google ADC, initially established with the supported Google authentication tooling. + +Each clone configures its remote in the repository-local Git config: + +```bash +git drs remote add anvil terra --checkout hydrate +``` + +Here `anvil` is the local remote name and `terra` is the built-in endpoint +alias. The alias supplies the Terra provider, production DRS endpoint, +Google ADC authentication, and read-only behavior; the unified command does +not use the legacy `terra` subcommand, `--drs-endpoint`, or `--mode` flags. +Repository-local Git config is authoritative. It is not tracked, and +credential material remains in the provider credential store. + +### First user: add and publish references + +For individual objects: + +```bash +git clone +cd + +gcloud auth application-default login + +git drs add-ref --remote anvil \ + drs:/// data/sample-1.cram +git drs add-ref --remote anvil \ + drs:/// data/sample-2.cram + +git add .gitattributes data/ +git commit -m "Add AnVIL data references" +git push +``` + +For multiple objects, the preferred POC workflow is a manifest: + +```bash +git drs add-ref --remote anvil --manifest references.tsv +git add .gitattributes data/ +git commit -m "Add AnVIL data references" +git push +``` + +The minimum manifest schema is: + +```text +drs_uri\tpath +drs:///\tdata/sample-1.cram +drs:///\tdata/sample-2.cram +``` + +Optional `size` and `sha256` columns can avoid redundant metadata work, but the command must validate supplied values against authoritative DRS metadata before writing pointers. + +### Second user: clone and hydrate + +```bash +gcloud auth application-default login + +git clone +cd +git drs pull +``` + +Selective hydration remains available: + +```bash +git drs pull -I "data/*.cram" +``` + +The second user does not need the first user's `.git/drs` directory, local Git configuration, cached objects, access tokens, or signed URLs. + +## Actual Blockers + +### 1. Terra runtime does not construct an authenticated data resolver + +Terra configuration and runtime selection exist, but the Terra runtime currently creates a context without the Syfon client used by the existing `add-ref` and pull implementations. Terra can therefore be configured and health-checked, but its object-resolution and download path is incomplete. + +The POC needs a real AnVIL resolver that authenticates with Google ADC and can retrieve object metadata and authorized access information. + +### 2. Data operations are coupled to the Syfon client + +Current reference and download paths call `GitContext.Client.DRS()` directly. This assumes every provider can be represented by the Syfon client and fails for the current Terra context. Provider-neutral resolution must replace direct Syfon coupling in `add-ref` and pull. + +### 3. `--remote-type` is exposed but does not select behavior + +`add-ref` exposes `--remote-type`, but resolver behavior should be selected from the configured remote. The normal interface should be `--remote anvil`; the command loads that remote, observes `type: terra`, and uses the AnVIL resolver automatically. + +If `--remote-type` remains temporarily, it must be hidden or deprecated and must be rejected when it conflicts with the selected remote. + +### 4. SHA256-bearing references can lose their DRS identity + +Current `add-ref` behavior writes a Git LFS-style SHA256 pointer when DRS metadata includes SHA256, while it writes a DRS URI pointer only when SHA256 is unavailable. It stores supplementary metadata under `.git/drs`, which is clone-local and is not transferred by Git. + +As a result, a fresh clone may receive a SHA256 and size but not the canonical AnVIL DRS URI needed to resolve the original record. The POC cannot depend on checksum search or the first user's `.git/drs` metadata. + +### 5. Pointer identity, cache identity, and content identity are conflated + +The current implementation often treats a pointer OID as both a cache key and content SHA256. AnVIL references require three explicit concepts: + +- canonical remote identity: the normalized DRS URI; +- local cache identity: a deterministic, filesystem-safe hash derived from the DRS URI; +- content identity: SHA256 or another durable checksum supplied by DRS metadata. + +### 6. Current DRS URI resolution is not AnVIL-aware + +Deriving `https://` and constructing an anonymous generic client is insufficient for authenticated AnVIL resolution. The configured Terra resolver must own authority routing, authentication, object lookup, and access URL retrieval. + +### 7. Repository-level remote configuration is missing + +Git-local remote configuration is not cloned. Without committed public configuration, every consumer must manually reconstruct the same endpoint, authentication mode, and resolver type. The POC needs a tracked, non-secret configuration format with safe local overrides. + +### 8. There is no complete two-user AnVIL acceptance test + +Existing focused tests cover pieces of Terra configuration, pointer parsing, and ping behavior. A functional POC requires a clean-clone test using independent user state and real or contract-faithful AnVIL resolution. + +## Required Prototype Decision: Commit the Canonical DRS URI + +Every Terra/AnVIL reference **must commit the canonical DRS URI in the pointer file**, including when the DRS record supplies a SHA256 checksum. + +This is required because: + +- `.git/drs` metadata is local and is not pushed or cloned; +- a checksum is content metadata, not necessarily a resolvable AnVIL record identifier; +- the second user must be able to resolve the same record from Git state alone; +- signed access URLs are temporary and must never be committed. + +The initial pointer format is: + +```text +version https://calypr.github.io/spec/v1 +oid drs:/// +size +sha256 +``` + +If the pointer parser cannot initially support the optional checksum line, the minimum viable pointer is: + +```text +version https://calypr.github.io/spec/v1 +oid drs:/// +size +``` + +Pull may retrieve the checksum from DRS metadata before download. Under no circumstances may the presence of SHA256 cause Terra `add-ref` to omit the DRS URI from committed state. + +For local storage, derive a cache key without replacing canonical identity: + +```text +cache_oid = sha256("git-drs-anvil-ref:v1\n" + normalized_drs_uri) +``` + +## Implementation Plan + +### Phase 1: Define a provider-neutral resolver + +Introduce a provider-neutral interface used by both `add-ref` and pull: + +```go +type Resolver interface { + GetObject(ctx context.Context, drsURI string) (*ResolvedObject, error) + GetAccess(ctx context.Context, drsURI, accessID string) (*ResolvedAccess, error) +} + +type ResolvedObject struct { + DRSURI string + ID string + Name string + Size int64 + Checksums map[string]string + AccessMethods []AccessMethod +} + +type ResolvedAccess struct { + URL string + Headers map[string]string + ExpiresAt *time.Time +} +``` + +Implement a resolver factory selected by configured remote type: + +- `SyfonResolver` for existing Gen3/local behavior; +- `AnVILResolver` for Terra/AnVIL behavior. + +Resolver results must not be serialized to tracked files when they contain signed URLs, credentials, or authorization headers. + +### Phase 2: Implement AnVIL authentication + +Support only `auth: google-adc` for the POC. + +The implementation must: + +1. Obtain credentials from the current user's ADC environment. +2. Request the scopes required by the supported AnVIL resolver. +3. attach bearer tokens only to trusted resolver endpoints; +4. refresh tokens automatically; +5. redact authorization headers and signed URLs from logs and errors; +6. return distinct errors for absent credentials, expired credentials, and denied authorization. + +Tokens, refresh credentials, signed URLs, and request headers must never be written to Git configuration or repository files. + +### Phase 3: Implement AnVIL DRS URI resolution + +The AnVIL resolver must: + +1. Parse and validate `drs:///`. +2. Normalize and preserve the original URI as canonical identity. +3. Route the reference through the configured AnVIL/Terra resolver contract. +4. Retrieve object size, durable checksums, and supported access methods. +5. Resolve an authorized access URL only when a download begins. +6. distinguish not-found, unauthorized, resolver-unavailable, and unsupported-access-method failures. + +Authority routing must be implemented inside `AnVILResolver`; it must not assume every URI authority is itself an HTTPS GA4GH DRS endpoint. Exact production endpoints, OAuth scopes, resolver requests, and response contracts must be validated against current authoritative AnVIL/Terra documentation before implementation is considered complete. + +### Phase 4: Make Terra pointer files self-contained + +Change Terra `add-ref` to always write a DRS pointer, regardless of checksum availability. + +The command must: + +- preserve the normalized DRS URI; +- store expected size; +- store SHA256 when available; +- avoid writing signed access information; +- write pointers deterministically; +- reject destination paths outside the repository; +- avoid depending on clone-local `.git/drs` metadata for later resolution. + +Update parsing, inventory, clean/smudge handling, and status output to preserve the distinction between a DRS URI and a SHA256-shaped OID. + +### Phase 5: Separate remote, cache, and content identity + +Introduce an explicit identity value: + +```go +type ObjectIdentity struct { + DRSURI string + CacheOID string + ContentSHA256 string +} +``` + +Use: + +- `DRSURI` for object and access resolution; +- `CacheOID` for local cache fanout paths; +- `ContentSHA256` for integrity validation when available. + +Normalize DRS URIs before deriving `CacheOID`, version the derivation scheme, and add golden tests so future normalization changes do not silently orphan cached content. + +### Phase 6: Hydrate through the AnVIL resolver + +For each DRS pointer, pull must: + +1. Parse its canonical DRS URI, expected size, and optional checksum. +2. Derive its local cache OID. +3. reuse an existing cache entry only after validating it; +4. resolve current object metadata with the selected AnVIL remote; +5. request a fresh authorized access URL; +6. download to a temporary file with cancellation and bounded retry; +7. validate size and SHA256 when available; +8. atomically promote verified content into the cache; +9. hydrate the destination path; +10. leave the pointer and cache uncorrupted on failure. + +Expired signed URLs must be handled by resolving a new URL rather than persisting or reusing stale access information. + +### Phase 7: Add manifest-driven batch `add-ref` + +Add: + +```bash +git drs add-ref --remote anvil --manifest references.tsv +``` + +The implementation must: + +1. Support required `drs_uri` and `path` columns. +2. Optionally accept `size` and `sha256` as asserted metadata. +3. Validate the entire manifest before modifying the worktree. +4. Reject malformed URIs, absolute paths, repository escapes, duplicate paths, and conflicting entries. +5. Resolve metadata with a small bounded worker pool. +6. compare asserted size/checksum values with authoritative DRS metadata; +7. report all validation failures together where practical; +8. write deterministic pointers after successful validation; +9. support `--dry-run`; +10. print a deterministic result summary. + +The batch operation is a wrapper over the same single-reference primitive, not an independent resolution system. + +### Phase 8: Configure each clone + +Use the remote command to write the single authoritative repository-local Git configuration after cloning. + +Initial schema: + +```bash +git drs remote add anvil terra --checkout hydrate +``` + +This is the unified `[name] ` form. Do not reverse the +arguments: `git drs remote add terra anvil` is the deprecated +provider-specific command shape. + +Configuration rules: + +- Git config is the only remote configuration representation; +- credentials remain in environment variables or provider credential stores; +- every clone runs `git drs remote add` before pulling. + +## Git Push Semantics for This Prototype + +AnVIL data already exists and the configured Terra remote is read-only. Publishing references therefore uses ordinary Git: + +```bash +git add .gitattributes data/ +git commit -m "Add AnVIL data references" +git push +``` + +`git push` publishes only: + +- pointer files; +- repository paths and Git history; +- `.gitattributes` rules; +- an optional reference manifest when the repository chooses to track it. + +It does not: + +- upload payload bytes; +- create or update AnVIL DRS records; +- copy records into another DRS service; +- transfer user credentials; +- persist signed download URLs. + +`git drs push` against a read-only Terra remote must fail before performing work with an actionable message such as: + +```text +remote "anvil" is read-only; publish AnVIL references with ordinary git push +``` + +Plain `git push` must not attempt to contact AnVIL. Data authorization is evaluated later, independently for each user, when `git drs pull` resolves and downloads the committed references. + +## Prototype Acceptance Tests + +### 1. Single-reference two-user flow + +1. User A authenticates with ADC. +2. User A adds a stable AnVIL DRS reference. +3. The generated file is a pointer containing the canonical DRS URI. +4. User A commits and pushes with ordinary Git. +5. User B clones into a clean environment. +6. The cloned file remains a pointer. +7. User B authenticates independently. +8. User B runs `git drs pull -I `. +9. The file is hydrated and matches expected size and checksum. + +### 2. Multiple-reference manifest flow + +Import at least ten references, including: + +- small and multipart-sized objects; +- nested destination paths; +- records with SHA256; +- a record without SHA256, if the target test data provides one. + +Verify deterministic pointers, full pull, selective pull, and an accurate summary. + +### 3. Fresh-clone portability + +Before User B clones, ensure no User A state is copied, including: + +- `.git/drs`; +- `.git/lfs/objects`; +- local Git configuration; +- environment tokens; +- ADC files. + +Successful User B hydration proves that committed pointers and repository configuration are sufficient. + +### 4. Authorization isolation + +A user who can clone Git but cannot access the AnVIL objects must receive a clear authorization error. Confirm that no token, authorization header, or signed URL appears in Git history, pointer content, standard output, standard error, or logs. + +### 5. Missing and malformed references + +Verify actionable, distinct failures for: + +- malformed DRS URI; +- unknown record; +- unsupported authority; +- record without a supported access method; +- duplicate or escaping destination path. + +Batch validation should avoid partial worktree modification unless partial behavior is explicitly requested. + +### 6. Expired access URL + +Simulate or wait for access URL expiry. Pull must resolve a fresh URL and continue without relying on stored access information. + +### 7. Integrity failure + +Inject incorrect bytes or metadata. Pull must fail verification, delete temporary/corrupt cache content, and leave the worktree pointer intact. + +### 8. Interrupted download and retry + +Interrupt a download and retry it. The retry must either resume safely or restart cleanly, never treating partial bytes as a valid cache entry. + +### 9. Idempotency and cache reuse + +After successful hydration, repeated `git drs pull` must not redownload verified content. Two paths referencing the same canonical URI should reuse the same cache entry. + +### 10. Read-only push behavior + +Verify that: + +- ordinary `git push` succeeds without contacting AnVIL; +- `git drs push` against the Terra remote fails clearly and performs no registration, upload, or deletion. + +### 11. Repository configuration safety + +Verify that repository-local Git configuration: + +- is created by `git drs remote add` after a fresh clone; +- is the only source of remote metadata; +- contains credential source identifiers but never credential values. + +## Backlog in Priority Order + +### P0: required for the two-user POC + +1. Introduce the provider-neutral resolver abstraction. +2. Confirm the authoritative AnVIL resolver endpoint, request contract, OAuth scopes, and supported access methods. +3. Implement Google ADC authentication and token refresh. +4. Implement AnVIL object metadata and access resolution. +5. Make Terra `add-ref` always commit the canonical DRS URI. +6. Add explicit DRS URI, cache OID, and content checksum identities. +7. Refactor pull to use `AnVILResolver` rather than a Syfon client. +8. Validate size and SHA256 and atomically manage cache content. +9. Implement and safely load repository-local Git configuration. +10. Reject `git drs push` for read-only Terra remotes. +11. Add a real or contract-faithful two-user clone-and-pull acceptance test. + +### P1: required for a usable prototype + +1. Add manifest-driven batch `add-ref` and `--dry-run`. +2. Add bounded parallel metadata resolution and download. +3. Add safe retry, cancellation, and expired-URL re-resolution. +4. Add `git drs doctor` checks for ADC, configuration, endpoint health, resolution, authorization, and pointer validity. +5. Improve errors for absent credentials, denied access, missing records, invalid manifests, and integrity failures. +6. Add CI coverage for clean-clone setup, authorization isolation, and repository configuration safety. +7. Publish versioned binaries and a compatibility matrix for the tested AnVIL contract. + +### P2: explicitly deferred + +1. Uploading payloads to AnVIL. +2. Creating, editing, or deleting AnVIL DRS records. +3. Copying records to Syfon or Gen3. +4. Destructive delete and remote garbage collection. +5. Automatic Terra workspace-table synchronization. +6. Additional Google authentication modes. +7. Generalized support for every DRS authority or provider. +8. A broad multi-provider pointer-format redesign beyond what portable AnVIL references require. + +## Definition of Done + +The AnVIL/Terra reference POC is functional when: + +- User A can add at least ten AnVIL DRS references in one manifest command. +- Each generated pointer commits the normalized canonical DRS URI. +- Pointers also preserve expected size and SHA256 when available. +- The Git repository contains no payload bytes, credentials, authorization headers, or signed URLs. +- Each clone configures the read-only AnVIL remote in repository-local Git configuration. +- Ordinary `git push` publishes the references without contacting AnVIL. +- `git drs push` refuses to operate on the read-only Terra remote. +- User B can clone without receiving any User A clone-local metadata, cache, or credentials. +- User B can authenticate with their own Google ADC identity and run `git drs pull` successfully. +- Full and selective hydration both work. +- An unauthorized user can clone pointers but cannot hydrate controlled-access data. +- Every download is size-verified and SHA256-verified when a checksum is available. +- Interrupted, expired-URL, and integrity-failure paths do not poison the cache or overwrite pointers with invalid content. +- Repeated pulls are idempotent and reuse verified cache entries. +- The complete two-user flow runs as a documented acceptance test in a clean environment. diff --git a/docs/anvil.md b/docs/anvil.md new file mode 100644 index 00000000..836f39ff --- /dev/null +++ b/docs/anvil.md @@ -0,0 +1,142 @@ +# Use AnVIL data references with git-drs + +This guide is for AnVIL users who want to version a dataset layout in Git +without copying controlled-access data into Git. A Git commit contains stable +DRS references and public resolver settings only. Each person downloads data +with their own Google identity and AnVIL authorization. + +## Before you begin + +Install Git, `git-drs`, the Google Cloud CLI, and obtain access to both the Git +repository and the referenced AnVIL data. Establish Application Default +Credentials (ADC) for your user: + +```bash +gcloud auth application-default login +``` + +ADC is local user state. Never copy the ADC JSON file into the repository. + +## Configure an AnVIL repository + +Configure the remote in repository-local Git config: + +```bash +git drs remote add anvil terra --checkout hydrate +``` + +`anvil` is the chosen local name and `terra` is the built-in endpoint alias. +The alias selects the Terra provider, production endpoint, Google ADC, and +read-only behavior, so no provider subcommand, `--drs-endpoint`, or `--mode` +flag is needed. The command stores only public remote metadata in `.git/config`. +Google ADC remains local user state. Each clone must run the command because +`.git/config` is not tracked. + +## Publish one reference + +Use the configured remote and choose the path that the data should occupy: + +```bash +git drs add-ref --remote anvil \ + drs:/// data/sample.cram +# Run this only after add-ref finishes successfully. +git add .gitattributes data/sample.cram +git commit -m "Reference AnVIL sample" +git push +``` + +For a Terra remote, `add-ref` authenticates with your ADC, validates metadata, +and writes a small pointer. The pointer retains the canonical DRS URI even when +the record has a SHA256 checksum. It never contains an access token or signed +download URL. Destination paths must stay inside the Git repository. + +AnVIL remotes are read-only. `git drs push` refuses a Terra remote before doing +any remote work because that command uploads payloads to writable DRS +providers. The failure does not require backing out a commit that references +existing Terra data: publish that commit and its pointers with ordinary +`git push` to a Git remote. Plain `git push` transfers only the Git commit and +small DRS references; it does **not** upload the referenced files to Terra or +create or modify Terra DRS records. + +## Clone and hydrate as another user + +The consumer authenticates independently and then hydrates all pointers: + +```bash +gcloud auth application-default login +git clone +cd +git drs pull +``` + +Hydrate only selected paths with an include pattern: + +```bash +git drs pull -I "data/*.cram" +``` + +Cloning the Git repository does **not** grant AnVIL data access. A user without +permission can inspect reference paths and DRS URIs but receives an +authorization error when hydration resolves the object. + +## Undo an accidental change to downloaded data + +`git drs pull` makes files downloaded from a read-only remote read-only. If a +user nevertheless makes a downloaded file writable, changes it, stages it, and +commits it, the clean filter records the changed content as a new pointer. +`git drs push` will still refuse the operation because the Terra remote is +read-only; that error does not by itself identify the accidental edit. + +If the mistaken commit is the latest commit and has not been pushed, restore +the original DRS pointer from its parent and amend the commit: + +```bash +# Inspect the change before rewriting the commit. +git diff HEAD^ HEAD -- data/sample.cram + +# Restore the original pointer in both the index and working tree. +git restore --source=HEAD^ --staged --worktree -- data/sample.cram +git commit --amend --no-edit + +# Confirm that the committed file is a pointer again. +git show HEAD:data/sample.cram +git status +``` + +If the entire latest commit was a mistake and it contains no work that should +be kept, it can instead be removed with `git reset --hard HEAD^`. Inspect the +commit and working tree first because this discards all changes in both. + +If the mistaken commit has already been shared, do not rewrite shared history. +Restore the pointer in a corrective commit: + +```bash +git restore --source=HEAD^ --staged --worktree -- data/sample.cram +git commit -m "Restore AnVIL DRS reference" +git push +``` + +After either repair, hydrate the restored pointer again when local access to +the payload is needed: + +```bash +git drs pull -I "data/sample.cram" +``` + +Use ordinary `git push` to publish commits containing read-only AnVIL +references. Do not use `git drs push` for a Terra remote. + +## Security and troubleshooting + +* `google application default credentials are unavailable`: run + `gcloud auth application-default login` as the current user and retry. +* `not authorized to access AnVIL DRS object`: confirm that the ADC identity has + access to the controlled dataset. Do not ask another user to share ADC files. +* `AnVIL DRS object not found`: verify the committed URI and that the configured + resolver supports its authority. +* Do not commit `.git/drs`, `.git/lfs`, Google credential files, bearer tokens, + request headers, or resolved access URLs. Access URLs are temporary and are + resolved again at download time. + +For pointer details, see [Pointer files](pointer-files.md). For the prototype +design and acceptance criteria, see [AnVIL/Terra POC](anvil-terra-poc.md). diff --git a/docs/bucket-mapping.md b/docs/bucket-mapping.md index ef68dbe1..e4a42ab4 100644 --- a/docs/bucket-mapping.md +++ b/docs/bucket-mapping.md @@ -198,7 +198,8 @@ They only need: Then they can run: ```bash -git drs remote add gen3 production HTAN_INT/BForePC --cred ~/.gen3/credentials.json +git drs remote add production calypr --scope HTAN_INT/BForePC \ + --credential file:~/.gen3/credentials.json ``` That remote setup resolves the bucket mapping from the server-side scope configuration. diff --git a/docs/commands.md b/docs/commands.md index e9c8a324..e59a9530 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -35,46 +35,128 @@ Use this when you want explicit initialization or to repair repo-local hooks/con ## Remote Configuration -### `git drs remote add gen3 [remote-name] ` +### `git drs remote add [name] [flags]` -Add or refresh a Gen3-backed Syfon remote. +Add a DRS server with the unified remote command. The built-in aliases are +`calypr`, `terra`, `synapse`, and `cgc`; inspect their non-secret defaults with +`git drs preset list` or `git drs preset show `. + +Calypr/Gen3 and Terra currently have operational runtime adapters. Synapse and +CGC are catalog-only; execution commands reject them explicitly rather than +constructing a partially authenticated client. ```bash -git drs remote add gen3 [remote-name] --cred -git drs remote add gen3 [remote-name] --token +git drs remote add cgc --credential env:CGC_TOKEN +git drs remote add synapse --credential helper:synapse +git drs remote add terra --scope my-billing-project/my-workspace +git drs remote add https://drs.example.org --provider ga4gh --auth none +git drs remote add research https://gen3.example.org \ + --provider gen3 --scope PROGRAM/PROJECT \ + --auth provider-helper:gen3-profile --credential profile:research ``` -Notes: +For a preset, its alias becomes the default local name; for a URL, the name is +derived from the host. Use the two-argument form to choose a name explicitly. +`--credential` accepts `env:VARIABLE`, `file:PATH`, `helper:NAME`, +`profile:NAME`, or `stdin`; it never accepts an inline secret. Other options +are `--scope`, `--auth`, `--provider`, `--storage`, and `--checkout`. Checkout +mode is `pointers` or `hydrate`. + +Provider values are `auto`, `ga4gh`, `gen3`, `terra`, `cgc`, and `synapse`. +Built-in aliases resolve `auto` to their catalog provider; a raw HTTPS endpoint +must specify a concrete `--provider` before any repository state is modified. +Authentication values are `auto`, `none`, `bearer`, `basic`, `google-adc`, +`provider-helper`, or `provider-helper:`. Presets are +expanded once and the resolved endpoint, provider, authentication method, and +catalog version are saved, so a later release cannot silently redirect an +existing remote. Only HTTPS endpoints without embedded credentials are +accepted. + +The older `gen3`, `local`, and `terra` command shapes are deprecated, hidden +compatibility forms. New scripts should use the unified command. + +### `git drs preset list` / `git drs preset show ` + +Display the presets embedded in this release. `list` shows each alias, +provider, authentication default, endpoint, and catalog version. `show` displays +the same details vertically and, when present, the registry service ID. Presets +contain no credential or secret values. + +| Alias | Provider | Authentication default | +| --- | --- | --- | +| `calypr` | Gen3 | Gen3 profile provider helper | +| `terra` | Terra | Google Application Default Credentials | +| `synapse` | Synapse | bearer token | +| `cgc` | Cancer Genomics Cloud | bearer token | + +The legacy provider-specific `remote add gen3`, `remote add local`, and +`remote add terra` forms remain only as hidden, deprecated compatibility +commands. Do not use them in new instructions or scripts. + +### `git drs remote list` -- `remote-name` is optional; if omitted, the default remote name is used -- scope is always one positional argument: `organization/project` -- `--cred` imports a Gen3 credential file -- `--token` uses a temporary bearer token -- if repo-local `git-drs` setup is missing, this command bootstraps it -- bucket resolution is scope-driven; users normally do not provide `--bucket` +List configured `git-drs` remotes. -### `git drs remote add local ` +```bash +git drs remote list +``` -Add or refresh a local Syfon/DRS remote. +### `git drs ping [remote-name]` + +Show the effective `git-drs` remote configuration and verify that the remote responds. ```bash -git drs remote add local local-dev http://localhost:8080 HTAN_INT/BForePC -git drs remote add local local-dev http://localhost:8080 HTAN_INT/BForePC --username drs-user --password drs-pass +git drs ping +git drs ping anvil ``` -Notes: +What it checks: -- local mode can store HTTP basic auth for helper flows -- repo-local `git-drs` setup is also bootstrapped here when missing +- prints the selected remote, remote type, endpoint, scope, bucket, storage prefix, and auth mode +- runs a health check against the selected remote +- for Terra DRS remotes, pings the GA4GH DRS service-info endpoint at `/ga4gh/drs/v1/service-info` +- for Terra/AnVIL TDR-hosted data in production, use `https://data.terra.bio` as the endpoint; Terra also uses DRSHub for DRS URI resolution, but DRSHub is a resolver service rather than the GA4GH DRS service-info host +- for scoped Syfon-style remotes, verifies that the configured organization/project and bucket are visible and readable -### `git drs remote list` +Example Terra preset configuration and ping: -List configured `git-drs` remotes. +```bash +git drs remote add anvil terra --scope my-billing-project/my-workspace +git drs ping anvil +``` + +Terra credential configuration: + +- The `terra` preset selects `google-adc`, which means Application Default Credentials from the local Google Cloud environment. +- Do not put Google access tokens, refresh tokens, service-account JSON, or other secrets in repo-local Git configuration or in `--credential`. +- Configure ADC outside `git-drs`, for example with `gcloud auth application-default login` for an interactive user credential or by setting `GOOGLE_APPLICATION_CREDENTIALS` to a service-account key path in automation. +- At runtime, Terra-aware resolution obtains Google credentials from the ADC provider chain; credential material is not written to committed repository files. +- `git drs ping` checks the public DRS service-info endpoint without sending the ADC credential. Authenticated object resolution and access requests still use ADC. + +A successful Terra ping includes: + +```text +remote: anvil (default) +type: terra +endpoint: https://data.terra.bio +health: ok +``` + +Troubleshooting: + +- `no remote configuration found`: run `git drs remote list` and pass an existing remote name, or add one with `git drs remote add anvil terra`. +- endpoint errors: remove the remote and add it again with the `terra` preset or a complete HTTPS URL. +- `terra DRS service-info returned ...`: verify the server is up and that the base endpoint is correct. You can test the exact URL with `curl -i https://data.terra.bio/ga4gh/drs/v1/service-info`. +- network, DNS, or TLS errors: check VPN/proxy/firewall settings and retry with `GIT_CURL_VERBOSE=1 git drs ping ` for additional HTTP diagnostics from Git-adjacent workflows. + +For developers, the live Terra ping integration test is intentionally behind the `integration` build tag because it reaches the public Terra DRS service: ```bash -git drs remote list +go test -tags=integration ./cmd/ping -run TestIntegrationPingTerraDRSServer -count=1 ``` +Set `GIT_DRS_TERRA_DRS_ENDPOINT` to point the test at a different Terra DRS deployment. Terra's DRSHub resolver URL follows the `https://drshub.dsde-.broadinstitute.org/api/v4/drs/resolve` pattern used by `terra-notebook-utils`, but `git drs ping` needs the GA4GH DRS service base URL that exposes `/ga4gh/drs/v1/service-info`; for Terra production that service base URL is `https://data.terra.bio`. + ### `git drs remote remove ` Remove a configured `git-drs` remote. @@ -94,6 +176,14 @@ Set the default `git-drs` remote. git drs remote set production ``` +### Remote roles: primary and source authorities + +A repository can have multiple `git-drs` remotes. The configured default remote, `drs.default-remote`, or an explicit command-level remote is the **primary remote** for repository-scoped operations such as push, registration, checksum lookup, and provider inspection. + +A **source DRS authority/resolver** is the DRS service named by a `drs://...` URI or by source metadata recorded for a reference-first object. For `git drs add-ref`, the client resolves the input `drs://...` URI against that source authority/resolver using source credentials. The primary remote does not act as a proxy for other source DRS servers. + +When the source authority is also configured as one of the repository remotes, `git-drs` can use that remote's endpoint and credentials for the source request. When the source authority is not configured as a named remote, the source URI itself remains the retrieval identity; adding the reference still does not create or mutate a primary-remote DRS record. + ## Tracking and Local Inventory ### `git drs track ` @@ -190,6 +280,8 @@ Notes: ## Provider/Object Reference Workflows +For details on pointer file formats and lifecycle state, see [Pointer Files and Reference State](pointer-files.md). + ### `git drs add-url [path]` Create a pointer plus local DRS metadata for an object that already exists in provider storage. @@ -205,16 +297,21 @@ Notes: - object-key mode resolves against the configured bucket scope - explicit provider URL mode remains supported - `--scheme` is required for object-key mode +- when `--sha256` is omitted, the pointer uses a derived local/cache OID and source URL metadata remains the retrieval identity - registration happens later on `git drs push` ### `git drs add-ref ` -Add a local pointer file for an existing DRS object. +Add a local pointer file for an existing DRS object. If the source DRS object has a SHA256 checksum, the pointer can use that checksum; otherwise the pointer preserves the source `drs://...` URI directly so hydration can resolve by DRS identity instead of checksum lookup. ```bash git drs add-ref drs://example/object-id data/object.bin ``` +`add-ref` also adds an exact read-only `filter=drs` rule for the destination to +`.gitattributes`. Stage that file together with the pointer so local inventory +commands such as `git drs ls-files` can discover the reference. + ### `git drs query ` Query a DRS object by ID or checksum. diff --git a/docs/compact-identifier-auth-discovery.md b/docs/compact-identifier-auth-discovery.md new file mode 100644 index 00000000..ad4e21d6 --- /dev/null +++ b/docs/compact-identifier-auth-discovery.md @@ -0,0 +1,140 @@ +# DRS compact identifiers, remote routing, and authentication discovery + +## Summary + +`git-drs` currently treats a DRS URI as an object identifier, not as a +replacement for remote configuration. Remote configuration continues to store +a concrete HTTP endpoint and provider-specific settings. + +The authority in a `drs://` URI participates in resolution in some code paths, +but it is not universally treated as a directly reachable HTTPS hostname. This +distinction is important because a DRS authority can be an identifier namespace +rather than a DNS name. + +The GA4GH DRS service-info operation can be used as one input to service and +authentication discovery. `git-drs` currently probes service-info for Terra +health checks, but it does not yet infer or persist an authentication method +from the response. + +## Does a DRS compact identifier replace the configured remote? + +No. There are two separate concepts: + +- A DRS URI or compact identifier, such as `drs://drs.anv0:v2_...`, identifies + a DRS object and its authority. +- A configured `git-drs` remote contains the concrete HTTP endpoint and any + provider-specific configuration needed to contact the service. + +The current configuration model stores an endpoint or base URL for Gen3, +Terra, and local remotes. These values are persisted as +`drs.remote..endpoint` in repository configuration. + +The proposed unified remote interface may accept maintained aliases such as +`terra`, `cgc`, and `synapse`. These aliases would expand to concrete, +versioned endpoint presets; they are conveniences, not DRS object identifiers +or separate protocols. + +The local compact OID described in the DRS identity ADR is another distinct +concept. The DRS URI is the canonical remote object identity, while the derived +OID is only a compact, filesystem-safe key for pointers and cache storage. The +derived OID does not identify or configure the DRS server. + +## Is the `drs://` authority resolved as a hostname? + +Partially, depending on the URI form and resolution path. + +### Slash-form URI + +For a URI such as: + +```text +drs://source.example.org/object-1 +``` + +the `add-ref` resolution path: + +1. parses the URI authority; +2. looks for a configured remote whose endpoint host matches that authority; +3. uses that configured remote, including its authentication, when one matches; +4. otherwise derives `https://source.example.org` and attempts an anonymous + DRS object lookup. + +This fallback allows a complete slash-form DRS URI to direct resolution to its +source service without incorrectly sending it to the selected primary remote. + +### Compact colon-form URI + +For a compact identifier such as: + +```text +drs://drs.anv0:v2_example +``` + +the AnVIL resolver understands that the colon separates the authority from the +object ID; it is not a network port separator. It extracts `drs.anv0` as the +authority and `v2_example` as the object ID. + +The AnVIL resolver does not convert `drs.anv0` into an HTTPS endpoint. Instead, +it routes the identifier through the configured, trusted Terra/AnVIL resolver +endpoint. In this case the DRS authority is a namespace used by the resolver, +not necessarily a directly reachable DNS host. + +The direct `add-ref` source-host parser currently expects an object ID in the +URI path. Compact colon-form identifiers therefore do not use the same +authority-derived anonymous fallback as slash-form identifiers; they depend on +the provider-specific resolver path. + +### Recommended routing rule + +The authority should participate in routing, but `git-drs` should prefer a +trusted authority-to-endpoint mapping or a configured resolver. It should not +universally assume that every DRS authority can safely be contacted as +`https://`. + +## Can service-info determine how authentication works? + +The GA4GH DRS service-info operation is available at the conventional route: + +```http +GET /ga4gh/drs/v1/service-info +``` + +For Terra remotes, `git drs ping` already sends this request, treats a +successful response as a health check, and prints the service-info response. +The request currently uses the default HTTP client. + +`git-drs` does not currently use that response to: + +- select bearer, Basic, Google ADC, or provider-helper authentication; +- identify a credential source; +- persist a discovered authentication method; +- discover authentication for every remote type. + +Service-info can be one trusted discovery input, but it may not contain enough +information to configure authentication completely. A response might identify +the service without specifying all of the OAuth issuer, audience, scopes, +credential source, or provider-specific login workflow required by a client. + +The proposed `--auth auto` behavior therefore uses a deterministic precedence: + +1. an explicit credential whose type implies one authentication method; +2. the maintained default from a trusted endpoint alias; +3. authentication metadata returned by trusted service discovery; +4. one installed provider helper for the detected provider; +5. anonymous access only after an anonymous capability probe succeeds. + +If discovery is absent, conflicting, or advertises multiple usable methods, +remote setup should require an explicit authentication choice. It must not try +each locally available credential against the server. Once selected, the +authentication method should be persisted so normal commands have stable +network behavior; an explicit diagnostic command may rerun discovery. + +## Current behavior at a glance + +| Question | Current behavior | +| --- | --- | +| Does a DRS compact identifier replace the configured endpoint? | No. Remote endpoints remain in configuration. | +| Is the `drs://` authority used for routing? | Partially. Slash-form external references can derive an HTTPS endpoint; AnVIL compact identifiers use the configured trusted resolver. | +| Can `git-drs` call service-info? | Yes. `git drs ping` calls it for Terra remotes. | +| Does service-info automatically configure authentication? | No. Authentication discovery is proposed but not implemented. | +| Should every authority be treated as an HTTPS hostname? | No. Prefer a trusted mapping or resolver because an authority may be a namespace rather than a reachable host. | diff --git a/docs/developer-guide.md b/docs/developer-guide.md index facce1a7..bf65b133 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -67,7 +67,9 @@ The pre-commit cache is non-authoritative and safe to rebuild. The local DRS met ### Setup path -`git drs remote add gen3 ...` or `git drs remote add local ...` is the standard entrypoint for connecting a repository. +The provider-neutral `git drs remote add [name] ...` is the +standard entrypoint for connecting a repository. Provider selection, scope, +authentication, and credential sources are flags rather than subcommands. That path: diff --git a/docs/e2e-modes-and-local-setup.md b/docs/e2e-modes-and-local-setup.md index 36870659..593f04e5 100644 --- a/docs/e2e-modes-and-local-setup.md +++ b/docs/e2e-modes-and-local-setup.md @@ -81,7 +81,11 @@ TEST_STRICT_CLEANUP=true - HTTP basic auth via: - `TEST_LOCAL_USERNAME` + `TEST_LOCAL_PASSWORD`, or - `TEST_ADMIN_AUTH_HEADER="Authorization: Basic "` -- `git drs remote add local ... --username ... --password ...` stores local basic auth in repo config for credential-helper flows. +- The local-only harness still exercises the hidden, deprecated + `git drs remote add local ...` compatibility path because the unified command + intentionally accepts HTTPS endpoints only. `--username` and `--password` + in this test path store local basic auth for credential-helper flows; they + are not the public CLI recommended for new scripts. ## How wrapper scripts map to the main suites @@ -226,7 +230,8 @@ Cause: Check: - `TEST_SERVER_MODE=local` -- remote configured with `git drs remote add local ...` +- local test remote configured through the deprecated + `git drs remote add local ...` compatibility path described above - local auth creds provided if server requires basic auth. ### `401 Unauthorized` on `/data/upload/...` in local mode diff --git a/docs/getting-started.md b/docs/getting-started.md index da9d724e..c36d3c37 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -16,20 +16,41 @@ The most important distinction is: - `git pull` updates commits and checkout state - `git drs pull` hydrates tracked pointer files already present in the checkout -## The Setup Command +## Connect A Remote -The standard setup command is: +List and inspect the presets shipped with the current release: ```bash -git drs remote add gen3 production --cred ~/.gen3/credentials.json +git drs preset list +git drs preset show calypr ``` -That command: +Then add a named remote from a preset. For example, a scoped Calypr/Gen3 +remote using a credential file is: -- stores the remote configuration -- imports or refreshes credentials +```bash +git drs remote add production calypr --scope \ + --credential file:~/.gen3/credentials.json +``` + +This command: + +- expands the preset into a pinned endpoint, provider, authentication method, + and preset catalog version +- stores only the credential source, not an inline secret - bootstraps repo-local `git-drs` wiring when it is missing +The built-in presets are `calypr`, `terra`, `synapse`, and `cgc`. Calypr/Gen3 +and Terra have operational runtime adapters; Synapse and CGC are catalog-only +and execution commands reject them until their adapters are implemented. The local +remote name is optional: `git drs remote add calypr ...` derives the name +`calypr`. You can also connect an unlisted HTTPS endpoint directly: + +```bash +git drs remote add research https://drs.example.org \ + --provider ga4gh --auth none +``` + ## The Two Common Workflows ### Existing Repository @@ -37,7 +58,8 @@ That command: ```bash git clone cd -git drs remote add gen3 production --cred ~/.gen3/credentials.json +git drs remote add production calypr --scope \ + --credential file:~/.gen3/credentials.json git drs pull ``` @@ -47,7 +69,8 @@ git drs pull mkdir my-data-repo cd my-data-repo git init -git drs remote add gen3 production --cred ~/.gen3/credentials.json +git drs remote add production calypr --scope \ + --credential file:~/.gen3/credentials.json git drs track "*.bam" git add .gitattributes git commit -m "Configure tracked files" @@ -127,12 +150,18 @@ git drs push That is the supported delete flow for tracked `git-drs` objects. For the fuller decision tree, see [Removing Files](remove-files.md). -### Refresh credentials +### Change a credential source ```bash -git drs remote add gen3 production --cred /path/to/new-credentials.json +git drs remote remove production +git drs remote add production calypr --scope \ + --credential file:/path/to/new-credentials.json ``` +The unified command refuses to overwrite an existing remote. Remove and add it +again when its endpoint, preset, or credential source must change. Prefer a +refreshing helper or profile source when the provider supports one. + ## Read Next - [Commands Reference](commands.md) for exact command syntax diff --git a/docs/github-pages.md b/docs/github-pages.md new file mode 100644 index 00000000..b092347e --- /dev/null +++ b/docs/github-pages.md @@ -0,0 +1,106 @@ +# Publishing documentation with GitHub Pages + +The [`Publish documentation`](../.github/workflows/pages.yaml) workflow builds +the end-user documentation selected in `mkdocs.yml` and renders the +[`anvil-terra-poc-presentation.md`](anvil-terra-poc-presentation.md) Marp deck. +It publishes both outputs as one GitHub Pages site. Do not edit or commit the +generated HTML. The Pages build explicitly selects Marp's `bespoke` template; +that template includes the browser runtime used for slide paging and keyboard +navigation, unlike the non-interactive `bare` template. It renders the deck to +a temporary build directory and copies the finished standalone HTML into the +site after MkDocs runs, preserving the embedded runtime and presentation styles. + +## Enable GitHub Pages + +An administrator only needs to configure Pages once for the repository: + +1. Open the repository on GitHub and select **Settings**. +2. Select **Pages** under **Code and automation**. +3. Under **Build and deployment**, set **Source** to **GitHub Actions**. +4. If GitHub prompts you to enable Actions, open **Settings → Actions → + General**, allow GitHub Actions to run, and save the setting. +5. Open **Actions → Publish documentation**, select **Run workflow**, choose the + `main` branch, and select **Run workflow**. +6. When the workflow finishes, its `deploy` job shows the published site URL. + The URL also appears in **Settings → Pages**. + +The repository must permit the official actions used by the workflow. If the +organization restricts actions, allow these actions and their pinned major +versions: + +- `actions/checkout@v4` +- `actions/configure-pages@v5` +- `actions/setup-python@v5` +- `actions/upload-pages-artifact@v3` +- `actions/deploy-pages@v4` + +The workflow declares the required `pages: write` and `id-token: write` +permissions itself. No deploy key, personal access token, or `gh-pages` branch +is required. + +## Publish updates + +After Pages is enabled, push a change under `docs/`, `mkdocs.yml`, or the Pages +workflow to `main`. The workflow runs automatically, renders the selected +Markdown pages with MkDocs, renders the deck with the pinned Marp CLI version, +and deploys the combined site. A maintainer can also publish without a content +change by running the workflow manually from the **Actions** tab. + +The generated site includes: + +```text +/index.html +/quickstart/ +/getting-started/ +/commands/ +/pointer-files/ +/adding-s3-files/ +/remove-files/ +/troubleshooting/ +/anvil-terra-poc-presentation.html +``` + +If the repository's default publishing branch is not `main`, update the +`on.push.branches` value in `.github/workflows/pages.yaml` before relying on +automatic deployments. + +## Preview locally + +With Python, Node.js, and npm installed, stage the selected end-user pages and +build the same combined site as CI: + +```bash +python -m pip install mkdocs-material==9.6.14 +rm -rf .pages-docs _site +mkdir .pages-docs +cp docs/{index,quickstart,getting-started,commands,pointer-files}.md .pages-docs/ +cp docs/{adding-s3-files,remove-files,troubleshooting}.md .pages-docs/ +cp docs/*.png .pages-docs/ +npx --yes @marp-team/marp-cli@4.2.3 \ + docs/anvil-terra-poc-presentation.md \ + --html \ + --template bespoke \ + --output /tmp/anvil-terra-poc-presentation.html +mkdocs build --strict +cp /tmp/anvil-terra-poc-presentation.html \ + _site/anvil-terra-poc-presentation.html +python -m http.server --directory _site 8000 +``` + +Open `http://localhost:8000/` in a browser. Keep dependency versions in these +commands synchronized with `.github/workflows/pages.yaml` when upgrading the +documentation toolchain. + +## Troubleshooting + +- **The workflow does not start after a push:** confirm the change is on + `main` and touches `docs/**`, `mkdocs.yml`, or + `.github/workflows/pages.yaml`. +- **The deploy job reports that Pages is not enabled:** repeat the enablement + steps and confirm the Pages source is **GitHub Actions**, not **Deploy from a + branch**. +- **An action is blocked:** ask an organization owner to allow the five + official GitHub actions listed above. +- **The page is not immediately available:** wait for the `deploy` job to + finish and use the URL shown in that job rather than guessing the repository + URL. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..b7671758 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,35 @@ +# Git DRS Documentation + +`git-drs` brings Git-compatible versioning workflows to large data stored +behind Data Repository Service (DRS) providers. Use these end-user guides to +install the command, connect a repository, manage pointer-backed files, and +resolve common problems. + +## Start here + +- [Quick Start](quickstart.md) — install `git-drs`, configure a remote, and + complete the first push and pull workflow. +- [Getting Started](getting-started.md) — understand the Git and DRS mental + model and the most common repository workflows. +- [Commands Reference](commands.md) — look up the supported CLI commands and + their options. + +## Work with data + +- [Pointer Files and Reference State](pointer-files.md) — learn what Git + commits in place of large payloads. +- [Add Existing S3 Objects](adding-s3-files.md) — create references for data + that is already in provider storage. +- [Remove Files](remove-files.md) — remove tracked files and understand remote + reconciliation behavior. +- [Troubleshooting](troubleshooting.md) — diagnose setup, authentication, + tracking, push, and hydration problems. + +## AnVIL/Terra proof of concept + +View the [AnVIL/Terra DRS reference presentation](anvil-terra-poc-presentation.html) +for a visual overview of the reference-only workflow, architecture, security +boundary, and remaining production-validation work. + +The presentation is generated from Markdown during the Pages build; its HTML +output is not stored in the repository. diff --git a/docs/installation.md b/docs/installation.md index 582ce425..7941fa7d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -84,7 +84,7 @@ Then move or copy the built binary somewhere on your `PATH`. Installing the binary does not configure a repository. Repository-local setup now normally happens when you run: ```bash -git drs remote add gen3 ... +git drs remote add [name] [flags] ``` ## Read Next diff --git a/docs/multi-drs-reference-identity-adr.md b/docs/multi-drs-reference-identity-adr.md new file mode 100644 index 00000000..5ca681c1 --- /dev/null +++ b/docs/multi-drs-reference-identity-adr.md @@ -0,0 +1,172 @@ +# ADR 0005: Preserve source DRS identity for reference-first workflows across mixed DRS servers + +## Status + +Proposed + +## Context + +`git-drs` supports two different classes of object tracking: + +1. **Normal local-file tracking** + - The user tracks local bytes. + - The clean filter calculates the content SHA256. + - The Git pointer uses `oid sha256:`. + - Downstream hydration can use the default configured DRS remote to look up DRS records by SHA256. + +2. **Reference-first tracking** through `add-ref` and `add-url` + - The user references object content that already exists somewhere else. + - The repository may not have local payload bytes at reference creation time. + - The referenced object may live behind a DRS server or provider/storage URL that is not the default DRS server. + - Not every DRS server used by a repository can be assumed to support SHA256 lookup. + +Today several implementation paths still rely on a SHA256-shaped local object identity. This works for local-file tracking because content bytes are available and the default remote is expected to resolve SHA256 checksums. It is not sufficient for `add-ref` and `add-url` when the source server cannot resolve by SHA256 or when the source DRS URI is the only durable retrieval identity. + +The important distinction is: + +- **content SHA256** identifies bytes when known; +- **local pointer/cache OID** gives `git-drs` a compact cache key and Git LFS-compatible pointer shape; +- **source DRS URI** identifies the remote DRS record and may be the only reliable way to retrieve bytes from a non-default or non-checksum-indexed DRS server. + +A downstream user who clones the repository must be able to hydrate reference-first objects without relying on the default DRS server's checksum lookup when the original source server does not support it. + +## Decision + +Adopt a split identity model for all reference-first workflows: + +1. **Normal local-file tracking remains content-SHA256 driven.** + - The clean filter calculates the real content SHA256 from local bytes. + - Pointers for normal tracked files continue to use `oid sha256:`. + - The default `remote` DRS server is assumed to support SHA256 lookup for these objects. + - Downstream users hydrate these files by checksum lookup against the default remote. + +2. **`add-ref` must preserve the original DRS URI as canonical source identity.** + - `git drs add-ref ` must store the input DRS URI as first-class metadata. + - If the source DRS object provides a real SHA256, store it as content metadata. + - If the source DRS object does not provide SHA256, do not fabricate a content checksum. + - Use a SHA256-shaped local/cache OID only as local identity when needed. + - Downstream hydration must resolve by the stored DRS URI when checksum lookup is unavailable or not appropriate. + +3. **`add-url` must preserve a canonical retrievable source reference.** + - If the input is a DRS URI, store that DRS URI as canonical source identity. + - If the input is a provider URL such as `s3://...`, store that provider URL as source access metadata and preserve enough remote/provider context to retrieve it later. + - If a real SHA256 is supplied through `--sha256`, store it as content metadata. + - If SHA256 is unknown, derive only a local placeholder/cache OID; mark it as non-content-derived. + - Downstream hydration must use the stored source reference rather than assuming checksum lookup will work. + +4. **Do not use DRS-URI-derived SHA256 values as content checksums.** + - A hash of `drs://...` is a local/cache key, not a byte checksum. + - Download validation must compare against a real content SHA256 only when one is known. + - If only DRS URI identity is known, validation should use size and DRS/provider metadata where available, and should clearly report that content SHA256 validation was unavailable. + +5. **Pointer/cache compatibility remains SHA256-shaped internally.** + - Existing fanout paths and many pointer workflows expect 64-hex OIDs. + - Reference-first objects may therefore use a derived local OID such as: + +```text +derived_oid = sha256("git-drs-source-ref:v1\nsource_uri=\nremote=\n") +``` + + - This derived OID is only a local identifier. + - It must be recorded with metadata that states its origin, for example `oid_kind = "derived-source-uri"`. + +## Required metadata model + +Reference-first metadata must make the following fields explicit: + +```text +local_oid SHA256-shaped local pointer/cache identity +local_oid_kind content-sha256 | derived-source-uri | placeholder +source_uri original drs:// URI or provider URL +source_type drs | s3 | gs | https | other +source_remote configured remote name, remote type, or resolver profile +content_sha256 real payload SHA256 when known +size object size when known +access_methods provider or DRS access metadata when available +``` + +The pointer may remain Git LFS-shaped for compatibility: + +```text +version https://git-lfs.github.com/spec/v1 +oid sha256: +size +``` + +But the sidecar/local metadata must be committed or otherwise made available to downstream users when it is required for hydration. A SHA256-shaped pointer alone is insufficient for non-checksum DRS servers unless `` is a real content SHA256 and a configured DRS/index can resolve it. + +A git-drs-specific pointer that embeds the DRS URI remains valid as an alternate representation: + +```text +version https://calypr.github.io/spec/v1 +oid drs://example/object +size +``` + +If this representation is used, the hydration path must preserve the OID type and resolve by DRS URI, not by checksum. + +## Hydration behavior + +Hydration should choose the retrieval strategy from metadata, in this order: + +1. **Source DRS URI path** + - If metadata contains `source_type = drs` and `source_uri`, resolve the DRS object through `source_remote` or the recorded resolver profile. + - Request an access URL from that source DRS server. + - Download to the local cache path for `local_oid`. + - Verify real SHA256 only if `content_sha256` is known. + +2. **Provider URL path** + - If metadata contains a provider URL and credentials/resolver context, retrieve through that provider path. + - Verify real SHA256 only if `content_sha256` is known. + +3. **Checksum lookup path** + - If `local_oid_kind = content-sha256` or `content_sha256` is known, lookup by checksum on the configured remote. + - This is the normal path for local-file tracking and remains supported for DRS servers that provide checksum lookup. + +4. **Failure path** + - If the only available identity is a derived/placeholder OID and there is no source URI/provider metadata, hydration must fail with an actionable error explaining that the repository lacks enough source identity to retrieve the object. + +## `add-ref` implications + +`add-ref` should no longer reduce a DRS reference to only a SHA256 pointer. + +Required behavior: + +1. Resolve the input DRS URI against the selected remote/resolver. +2. Capture returned size, checksums, access methods, and DRS object ID when available. +3. Persist the original input DRS URI as `source_uri`. +4. Persist the selected remote or resolver as `source_remote`. +5. Use real content SHA256 as `content_sha256` only if the DRS object reports it. +6. Use either: + - `local_oid = content_sha256` when a real SHA256 is known and policy permits checksum identity; or + - `local_oid = sha256(namespaced normalized source_uri + remote context)` when SHA256 is missing or when the source server cannot support checksum lookup. +7. Ensure downstream hydration resolves by `source_uri` when `source_remote` does not support SHA256 lookup. + +## `add-url` implications + +`add-url` should treat its input URL as source identity, not merely as a seed for a placeholder OID. + +Required behavior: + +1. Inspect the source object for size and metadata where possible. +2. Preserve the input object URL as `source_uri` or provider source metadata. +3. Preserve the selected remote/provider resolver context. +4. Use `--sha256` as real `content_sha256` only when supplied by the user or verified from trusted provider metadata. +5. If SHA256 is unknown, derive a local OID from namespaced source metadata and mark it as non-content-derived. +6. Ensure downstream hydration retrieves from the stored source URI/provider metadata rather than assuming checksum lookup. + +## Consequences + +- Normal local-file tracking remains compatible with existing SHA256 lookup behavior. +- `add-ref` and `add-url` become safe for multi-DRS repositories where some servers cannot look up by SHA256. +- Downstream users can hydrate reference-first objects because the original source identity is preserved. +- Cache paths remain compact and SHA256-shaped without confusing derived local IDs for payload checksums. +- Download verification must become metadata-aware: real SHA256 validation is required when known, but a derived local OID must never be treated as a byte checksum. +- Repository metadata must become part of the clone/hydration contract for reference-first objects. + +## Non-goals + +- Do not require every DRS server to implement SHA256 lookup. +- Do not require downloading payload bytes during `add-ref` or `add-url` solely to compute SHA256. +- Do not treat hashes of DRS URIs or provider URLs as real content checksums. +- Do not remove the existing SHA256-based workflow for normal local-file tracking. diff --git a/docs/pointer-files.md b/docs/pointer-files.md new file mode 100644 index 00000000..43e8d0bd --- /dev/null +++ b/docs/pointer-files.md @@ -0,0 +1,238 @@ +# Pointer Files and Reference State + +`git-drs` keeps large payload bytes out of Git history by committing small pointer files. A pointer records enough identity to let `git-drs` find or rebuild the payload later, while the payload itself lives in local cache, provider storage, or behind a DRS server. + +This page documents the pointer shapes used by reference-first workflows and how state changes as commands create references, download content, and obtain temporary access URLs (the resolved "real" download URLs used for the transfer). + +## Pointer file shapes + +### Content SHA256 pointer + +When the real payload SHA256 is known, `git-drs` uses a Git LFS-compatible pointer: + +```text +version https://git-lfs.github.com/spec/v1 +oid sha256: +size +``` + +For normal tracked-file workflows, `` is the real content SHA256. For metadata-first workflows, it may be a SHA256-shaped local/cache OID only when separate metadata preserves the retrievable source identity. + +### DRS URI pointer + +When `add-ref` resolves a DRS object that does not provide a real SHA256, `git-drs` preserves the source identity directly in a git-drs pointer: + +```text +version https://calypr.github.io/spec/v1 +oid drs:/// +size +``` + +The DRS URI is the retrieval identity. It is not a content checksum. Cache paths can still use a deterministic SHA256-shaped key derived from the DRS URI, but validation must not compare downloaded bytes against that derived key as if it were a content SHA256. + +## State model + +The same path can move through these states: + +| State | Worktree file | Local cache | Source metadata | Retrieval behavior | +|---|---|---|---|---| +| Pointer only | Pointer text is present at the tracked path | Payload may be absent | DRS URI, provider URL, or checksum metadata identifies the source | `git drs pull` can hydrate the path later | +| Cached payload | Pointer text remains in Git/index history, and payload bytes exist in `.git/lfs/objects/...` | Payload bytes are present | Same source metadata remains available | checkout/smudge can replace pointer text with payload bytes | +| Hydrated worktree | The tracked path contains payload bytes | Payload bytes are present | Source metadata remains the retrieval contract | Future clean/filter operations can write the pointer back to Git | + +A DRS/provider access URL is usually not stable source identity. It is a temporary, resolved "real" download URL created from the stable DRS URI or provider reference when content is downloaded. + +## Command-created state changes + +### `git drs add-ref ` + +`add-ref` creates a reference to an existing DRS object without downloading payload bytes solely to compute a checksum. + +1. The client resolves the input DRS URI against the source authority/resolver named by the `drs://...` URI, using credentials appropriate for that source. +2. If the DRS object reports a real SHA256, `git-drs` writes a SHA256 pointer whose OID is that content checksum. +3. If the DRS object does not report a real SHA256, `git-drs` writes a DRS URI pointer whose OID preserves the original `drs://...` source reference. +4. The fetched DRS metadata is stored locally under a deterministic local/cache OID. +5. The worktree path is in the **pointer only** state until content is hydrated. + +Example pointer when SHA256 is unavailable: + +```text +version https://calypr.github.io/spec/v1 +oid drs://example.org/object-1 +size 123456 +``` + +### `git drs add-url [path]` + +`add-url` creates a reference to an object that already exists in provider storage. + +1. The command inspects the provider object through the configured remote. +2. The input provider URL or resolved provider URL is preserved as source access metadata. +3. If `--sha256 ` is supplied or a trusted SHA256 is discovered, that value is stored as content metadata and used as the pointer/cache OID. +4. If SHA256 is unknown, `git-drs` derives a placeholder/local OID from source metadata and writes a pointer with that SHA256-shaped local OID; it does **not** record the placeholder as a content checksum. +5. The worktree path is in the **pointer only** state until content is hydrated. + +Example pointer when SHA256 is unknown: + +```text +version https://git-lfs.github.com/spec/v1 +oid sha256: +size 123456 +``` + +In that case, `` is a local/cache identifier, not a payload checksum. + +### `git drs pull` + +`pull` changes pointer-only paths into hydrated content when payload bytes are missing locally. + +1. The command scans tracked pointer files in the current checkout. +2. For a content SHA256 pointer, it can look up a scoped DRS record by checksum and request a temporary access URL from that DRS record. +3. For a DRS URI pointer, it resolves the stored `drs://...` source URI, requests a temporary access URL for that DRS object, and downloads through the resolved URL. +4. For provider-backed references, it uses the stored provider/source metadata to retrieve content when available. +5. Downloaded bytes are written to the local cache path. +6. If a real content SHA256 is known, the downloaded bytes are validated against it. If only a DRS URI or derived local OID is known, validation is limited to size and available DRS/provider metadata. +7. The worktree file is checked out from the cache, moving it to the **hydrated worktree** state. + +The temporary access URL (the resolved "real" URL) created during this process is a download mechanism, not the durable identity committed in Git. The durable identity remains the pointer plus source metadata: content SHA256 when known, DRS URI for DRS references, or provider URL/source metadata for provider references. + +## Primary remote DRS server state + +The **primary remote DRS server** is the configured `drs.default-remote` or the remote passed to a command. For reference-first flows, distinguish three places where state can exist: + +- **Git state:** committed pointer text at the repository path. +- **Local git-drs state:** sidecar DRS object JSON under the repo's local DRS metadata directory, plus optional local LFS cache bytes. +- **Primary remote DRS server state:** Syfon/Gen3 DRS records, scoped to the configured organization/project, and any remote storage object those records point at. + +`add-ref` and `add-url` are intentionally metadata-first commands. They create Git and local git-drs state first; they do not necessarily create or mutate the primary remote DRS server until a later `git drs push` or an explicit server-side copy/register operation. + +### Sequence: pointer and DRS server state for `add-ref` / `add-url` + +The following sequence diagram shows the expected pointer-file changes, local +metadata/cache changes, and primary remote DRS server changes for the +reference-first `add-ref`/`add-url` flows. If a caller says `add-reg` in this +context, treat it as the registration side of the same reference-first flow: +the server-side record is created or refined only when metadata is pushed or +explicitly registered, not when the local pointer is first written. + +```mermaid +sequenceDiagram + autonumber + actor User + participant Git as Git worktree/index + participant Local as Local git-drs metadata/cache + participant Source as Source DRS authority/resolver + participant Remote as Primary remote DRS server + participant Provider as Provider object + + rect rgb(238, 248, 255) + note over User,Source: add-ref: reference an existing DRS URI + User->>Source: git drs add-ref drs://source/object data/file + Source-->>User: DRS object metadata, size, access methods, optional sha256 + note over User,Source: Client resolves drs://source/object against the source authority/resolver using source credentials. + alt Resolved object includes real sha256 + User->>Git: Write pointer oid sha256: + User->>Local: Store DRS metadata keyed by real sha256 + else Resolved object does not include real sha256 + User->>Git: Write git-drs pointer oid drs://source/object + User->>Local: Store DRS metadata keyed by derived DRS/local cache key + end + note over Remote: No new scoped DRS record is required merely because add-ref wrote a local pointer. + end + + rect rgb(248, 255, 238) + note over User,Provider: add-url: reference an existing provider object + User->>Remote: git drs add-url s3://bucket/key data/file [--sha256] + Remote->>Provider: Inspect provider object using remote bucket credentials/scope + Provider-->>Remote: Provider URL, size, ETag/metadata, optional trusted sha256 + Remote-->>User: Inspected object metadata + alt Real sha256 supplied or trusted + User->>Git: Write pointer oid sha256: + User->>Local: Store local DRS metadata with checksum sha256= + else Real sha256 unknown + User->>Local: Calculate placeholder/local OID from provider metadata + User->>Git: Write pointer oid sha256: + User->>Local: Store provider access URL and size without checksum= + end + note over Remote: Inspection does not itself create the durable DRS record for this repo scope. + end + + rect rgb(255, 248, 238) + note over User,Remote: Push/register metadata with primary remote + User->>Remote: git drs push / register metadata for reachable pointers + alt Pointer/local metadata has real sha256 + Remote->>Remote: Upsert scoped DRS record with sha256, size, and access method + else Only DRS URI/provider source identity is known + Remote->>Remote: Upsert metadata preserving source URI/provider access, size, and no fake sha256 + end + end + + rect rgb(255, 238, 248) + note over User,Source: Later hydration discovers real content sha256 + User->>Source: git drs pull resolves DRS URI or source metadata + Source-->>Local: Temporary access URL / downloaded payload bytes in cache/worktree + Local->>Local: Calculate real sha256 over downloaded bytes + Local->>Local: Store/refine local checksum metadata sha256= + opt User intentionally rewrites pointer metadata + User->>Git: Replace DRS/placeholder pointer with oid sha256: + end + opt User pushes refined metadata + User->>Remote: git drs push / register refined checksum metadata + Remote->>Remote: Store real sha256 as checksum, keep DRS/provider retrieval identity + Remote->>Remote: Ensure placeholder or derived local OID is not advertised as checksum + end + end +``` + +### After `git drs add-ref ` + +Expected state immediately after `add-ref` succeeds: + +| Location | Expected state | +|---|---| +| Git/worktree | A pointer file exists at ``. If the resolved DRS object has a real SHA256 checksum, the pointer uses `oid sha256:`; otherwise it uses a git-drs DRS URI pointer that preserves the input DRS URI. | +| Local git-drs metadata | The resolved DRS object returned by the selected remote/resolver is stored locally under the local OID used for the pointer/cache lookup. If the DRS object omitted `self_uri`, the input DRS URI is retained as `self_uri`. | +| Local payload cache | No payload download is required just to create the reference. The cache may still be empty for this object. | +| Primary remote DRS server | No new record is required merely because `add-ref` ran. The client should resolve the source DRS URI through the source authority/resolver using source credentials for future hydration. `add-ref` should be treated as creating a local reference to an existing DRS record, not as copying or registering that record into a new project scope. | + +If the input DRS URI points at another DRS authority or resolver, the primary remote's durable state after `add-ref` is still unchanged. The committed reference remains valid because the pointer/local metadata preserve the source DRS URI and hydration uses that source identity directly. + +### After `git drs add-url [path]` + +Expected state immediately after `add-url` succeeds: + +| Location | Expected state | +|---|---| +| Git/worktree | A Git LFS-shaped pointer file exists at `[path]`. With `--sha256`, its OID is the real content SHA256. Without `--sha256`, its OID is a deterministic SHA256-shaped placeholder/local OID derived from inspected provider metadata. | +| Local git-drs metadata | A local DRS object is written with the resolved provider URL as an access method. When `--sha256` is provided, the local DRS object includes a `sha256` checksum. When SHA256 is unknown, the placeholder/local OID is **not** written as a checksum. | +| Local payload cache | No payload upload or download is required. The object bytes are expected to already exist at the provider URL inspected by the remote. | +| Primary remote DRS server | No DRS record is required immediately. On a later `git drs push`, git-drs can bulk-register metadata for the pointer into the primary remote scope. For known-SHA objects, the remote record should include the real SHA256 and size. For unknown-SHA objects, the remote record must preserve the provider access URL and size, and must not claim that the placeholder/local OID is a real content checksum. | + +A primary remote may already know about the provider object because of its bucket credential/scope configuration. That inspection ability is not the same thing as a durable DRS record for the repository object; durable registration is a push-time concern. + +### After a DRS URI is downloaded and a real SHA256 is calculated + +Downloading through a DRS URI has two separate effects: + +1. It hydrates local state by resolving the stable DRS URI to a temporary access URL and writing payload bytes into the local cache/worktree. +2. It may reveal the real content SHA256 if the downloaded bytes are hashed locally. + +Expected state after the real SHA256 is known: + +| Location | Expected state | +|---|---| +| Git/worktree | Existing committed pointer identity should not silently change during download. A DRS URI pointer remains a DRS URI pointer until the user/tool intentionally rewrites and commits it as a content-SHA pointer. | +| Local payload cache | The downloaded bytes exist in the local cache. If a previous cache key was derived from a DRS URI or placeholder, that key remains a local retrieval/cache key, not proof of content identity. Implementations may also add an alias/cache entry keyed by the real SHA256, but should avoid losing the source DRS URI mapping. | +| Local git-drs metadata | The real SHA256 can be added as checksum metadata alongside the preserved DRS URI/provider source identity. The DRS URI/provider URL remains the durable retrieval contract unless and until a remote record with the real checksum is registered. | +| Primary remote DRS server | Merely downloading and hashing bytes locally should not mutate the primary remote. If the repository later pushes an updated metadata record, the primary remote should have a scoped DRS record whose `sha256` is the real content checksum, whose `size` matches the downloaded payload, and whose access methods still resolve to the original DRS/provider source or to a managed copied object. The remote should not retain a placeholder/local OID as a checksum once the real SHA256 is known. | + +When a placeholder or DRS-derived local OID is replaced in metadata by a real SHA256, treat it as a metadata refinement, not as evidence that the original source identity was wrong. The safe final server state is: **real content SHA256 for checksum-based lookup, stable DRS URI/provider access metadata for retrieval, and no placeholder/local OID advertised as a content checksum.** + +## Practical review guidance + +When reviewing pointer changes: + +- `oid sha256:<64-hex>` means either a real content SHA256 or a SHA256-shaped local OID; check source metadata when the object came from `add-url` without `--sha256`. +- `oid drs://...` means the pointer itself preserves the source DRS URI and hydration should resolve by DRS URI, not by checksum lookup. +- A hash derived from a DRS URI or provider URL must not be used as a content checksum. +- Temporary access URLs may be created during hydration, but they should not replace the durable source identity in committed pointer state. diff --git a/docs/quickstart.md b/docs/quickstart.md index f8411aed..97074351 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -56,9 +56,23 @@ Verify: git-drs version ``` -## 2. Get Credentials +## 2. Choose A Preset And Credential Source -Download your Gen3 API credentials JSON from your commons profile page and save it somewhere stable, for example: +See the non-secret server defaults included with your installed release: + +```bash +git drs preset list +git drs preset show calypr +``` + +The built-in aliases are `calypr`, `terra`, `synapse`, and `cgc`. Only +Calypr/Gen3 and Terra currently have operational runtime adapters; Synapse and +CGC are catalog-only presets. Presets supply +the endpoint, provider adapter, and usual authentication method; they never +contain credentials. + +For the Calypr/Gen3 workflow below, download your Gen3 API credentials JSON +from your commons profile page and save it somewhere stable, for example: ```bash ~/.gen3/credentials.json @@ -72,12 +86,18 @@ Typical flow: 4. Download the JSON file. 5. Save it somewhere stable. +On the command line, refer to the file as `file:~/.gen3/credentials.json`. +Other supported credential sources are `env:VARIABLE`, `helper:NAME`, +`profile:NAME`, and `stdin`. Do not put a token or password directly in +`--credential`. + ## 3. Connect An Existing Repository ```bash git clone cd -git drs remote add gen3 production --cred ~/.gen3/credentials.json +git drs remote add production calypr --scope \ + --credential file:~/.gen3/credentials.json git drs pull ``` @@ -89,12 +109,17 @@ Use this path when the repository already contains tracked pointers and you want mkdir my-data-repo cd my-data-repo git init -git drs remote add gen3 production --cred ~/.gen3/credentials.json +git drs remote add production calypr --scope \ + --credential file:~/.gen3/credentials.json git drs track "*.bam" git add .gitattributes git commit -m "Configure tracked files" ``` +`production` is the local remote name and `calypr` is the preset. If the preset +name is also a suitable local name, the shorter form is +`git drs remote add calypr ...`. + ## 5. Day-One Commands Update Git history: diff --git a/docs/remote-cli-design.md b/docs/remote-cli-design.md new file mode 100644 index 00000000..1175fca6 --- /dev/null +++ b/docs/remote-cli-design.md @@ -0,0 +1,400 @@ +# Remote CLI and multi-DRS server design + +## Status + +Implemented for built-in presets and unified endpoint configuration. Registry +discovery, custom user/system preset catalogs, capability probing, and remote +migration remain future work. + +## Problem + +`remote add` currently exposes the implementation as three different command +shapes: + +```text +git drs remote add gen3 [remote-name] +git drs remote add local +git drs remote add terra --drs-endpoint URL --auth google-adc --mode read-only +``` + +This creates several avoidable distinctions: + +- the provider is sometimes a command and sometimes effectively a deployment + choice; +- the endpoint is inferred, positional, or a flag depending on the provider; +- `--cred`, `--token`, `--username`, and `--password` encode credential formats + rather than a credential source; +- `--mode read-only` and `--auth google-adc` ask users to state the only values + currently accepted; +- `--bucket` mixes DRS resolution with Gen3 storage and publishing policy; +- `--no-skip-smudge` is repository checkout policy, not a remote property; and +- `local` describes where a server is deployed, not which DRS protocol it + implements. + +Adding Cancer Genomics Cloud (CGC), Synapse, or another GA4GH DRS service by +copying this pattern would produce another provider subcommand and another set +of slightly different flags for every service. + +## Proposed vocabulary + +Use one command whose only required concept is a server endpoint (or a +well-known endpoint alias). The local remote name should normally be derived +from the alias or endpoint host; an explicit name is only needed to override +that default or to configure the same service more than once. + +```text +git drs remote add [flags] +git drs remote add [flags] # explicit-name form + +Flags: + --scope Optional provider-specific collection/project scope + --auth auto, none, bearer, basic, google-adc, or provider-helper[:name] + --credential Credential source, never an inline secret + --provider auto, ga4gh, gen3, terra, cgc, or synapse + --storage Advanced write/registration storage override + --checkout pointers or hydrate +``` + +For example: + +```sh +# Presets supply the endpoint, provider, and authentication method. +git drs remote add cgc --credential env:CGC_TOKEN + +# A GA4GH Service Registry ID can select a service without spelling its URL. +git drs remote add registry:com.sb.cgc.drs --credential env:CGC_TOKEN + +git drs remote add synapse --credential helper:synapse + +# A preset may supply a public endpoint and authentication default. +git drs remote add terra --scope my-billing-project/my-workspace + +# A custom deployment needs its URL; its name defaults from the host. +git drs remote add https://example-gen3.org \ + --provider gen3 --scope PROGRAM/PROJECT \ + --credential file:~/.gen3/credentials.json +``` + +`` should be an HTTPS URL by default. Aliases such as +`terra`, `cgc`, and `synapse` are conveniences that expand to versioned, +maintained presets, including the endpoint and usual authentication method; +they are not separate protocols. Thus `cgc` is not repeated as both a name and +a URL, and users do not have to restate `--auth bearer` when the preset already +knows it. For a URL, derive a stable local name from its host and report it +before saving. If that name already exists, ask for an explicit name rather +than silently replacing it. `--provider` is an escape hatch when discovery is +absent or ambiguous, and should normally remain `auto`. + +### Preset catalog and overrides + +The built-in catalog is embedded from `internal/presets/presets.yaml` and can +be inspected with `git drs preset list` and `git drs preset show `. +Legacy provider-specific add commands remain hidden compatibility shims. + +When this proposal is implemented, the built-in preset list should be +maintained in a reviewed, version-controlled catalog in this repository. The +proposed location is `internal/presets/presets.yaml`; choosing that path in this +design does not create the file or implementation. The catalog should be +embedded in each `git-drs` release. Each entry should contain only non-secret +defaults: alias, endpoint, provider, authentication method, and optional +registry service ID. The catalog should not be downloaded at command runtime. +The proposed `git drs preset list` and `git drs preset show ` commands +should display the shipped entries, their source (`built-in`, `system`, or +`user`), and the release version that supplied them. + +Preset expansion should be a one-time operation during `remote add`. The +implementation should persist its resolved endpoint, provider, authentication +method, preset alias, and catalog version in the remote configuration, and +print those values before saving. Existing remotes would therefore not change +when a later `git-drs` release edits a preset. The proposed +`git drs remote diagnose ` may report a newer preset and offer an explicit +migration; it must not adopt one automatically. + +The proposed commands would allow users and administrators to add presets in +user and system configuration: + +```sh +git drs preset add lab https://drs.example.edu --provider gen3 --level user +git drs preset add institute https://drs.example.org --level system +``` + +Custom names must not silently shadow built-in aliases. A collision should +fail and direct the user to a qualified selector such as `user:lab` or +`system:lab`; built-ins may be selected explicitly as `built-in:cgc`. This +preserves a short, predictable `cgc` path while allowing local catalogs without +creating a route for configuration to redirect credentials unexpectedly. + +Explicit command-line values take precedence over non-endpoint preset defaults, +so `--auth`, `--provider`, and `--scope` can override the expanded values after +the command prints the effective configuration. To use a different endpoint, +the user should pass that HTTPS URL or a qualified custom preset instead of +overriding the meaning of a built-in alias. Credential sources and secret +values never belong in any preset catalog. + +### Registry discovery + +The [GA4GH Service Registry](https://registry.ga4gh.org/) can remove the need +to type an endpoint for services it lists. Prefer its service `id` as the +selector because that value is intended as an identifier: + +```sh +git drs remote add registry:com.sb.cgc.drs +git drs remote add registry:org.sagebase.prod.repo-prod +``` + +The `registry:` prefix makes network lookup explicit and avoids confusing a +registry selector with a built-in alias, local remote name, or URL. On a +successful lookup, persist the resolved URL and registry ID in the remote +configuration and print the registry name and URL for confirmation. Normal +operations, including `push`, use that persisted endpoint; they must not +silently follow a later registry change on every invocation. An explicit +`remote diagnose` or refresh operation may report a changed registry record +and ask the user whether to adopt it. + +A registry display `name` may be accepted as a convenience only when it has a +single exact match. It is not a reliable primary key: names are human-readable, +can change, can contain spaces, and are not necessarily unique. For example, +the registry currently contains two DRS records named `Canine Data Commons`. +Do not use fuzzy or prefix matching to guess a credential destination. If a +name matches zero or multiple records, list the candidate IDs and require the +user to select an ID or URL. Short forms such as `cgc` remain maintained aliases +rather than fuzzy registry-name searches. + +Consequently, an arbitrary local remote name in a fresh repository cannot by +itself safely determine a server. It can do so only when it is an exact +maintained alias or an explicit, uniquely resolved registry selector. + +Registry lookup discovers a candidate DRS resolver endpoint, not a publishing +contract. Registry records can represent multiple logical datasets at one URL, +and their URLs vary between service roots and paths ending in +`/ga4gh/drs/v1/objects`. Normalize and validate the advertised endpoint through +service-info and capability probing before saving it. In particular, a +registry match must not make `push` assume that upload or registration is +supported; the publisher capability checks described below still apply. If +the registry is unavailable or the record is invalid, fail with an actionable +request for an explicit URL rather than guessing. + +Neither a GA4GH Service Registry record nor the GA4GH service-info response +declares which HTTP `Authorization` scheme or credential flow a DRS deployment +requires. Registry discovery therefore supplies only service identity and a +candidate URL; service-info can confirm service identity and implementation +metadata, but it cannot select `bearer`, `basic`, `google-adc`, or a provider +helper. Do not infer authentication from registry membership or a DRS version. + +### Authentication methods + +`--auth` selects **how an HTTP request is authorized**. It is distinct from +`--credential`, which selects **where the material needed by that method comes +from**. For example, `--auth bearer --credential env:CGC_TOKEN` says to put a +bearer token in the request and to read that token from `CGC_TOKEN`. + +The methods have the following meanings: + +| Method | Meaning | +| --- | --- | +| `auto` | Select a method from trusted configuration and discovery, using the deterministic order below. This is the default; it does not mean "try every credential." | +| `none` | Send no authentication. If the server rejects the request, report that authentication is required rather than silently trying local credentials. | +| `bearer` | Send an OAuth-style bearer access token obtained from `--credential`. | +| `basic` | Send an HTTP Basic username/password pair obtained from `--credential`. | +| `google-adc` | Use Google Application Default Credentials to obtain and refresh the token expected by the service. | +| `provider-helper` | Delegate token acquisition and refresh to the adapter for the selected provider; the adapter returns request credentials, but does not replace the DRS resolver. | + +`auto` should resolve once when the remote is added, persist the selected +method (not a secret), and show the result to the user. The selection order is: + +1. an explicit `--credential` whose type implies a method, if unambiguous; +2. the endpoint alias's maintained default (for example, a Terra alias may + select `google-adc`); +3. a single installed provider helper registered for the detected provider; +4. `none` only when an anonymous capability probe succeeds. + +If the evidence is absent, conflicting, or names multiple usable methods, +`remote add` should stop and ask the user to choose `--auth`; it must not send +available credentials to a server merely because they exist. A 401 challenge +with a standards-compliant `WWW-Authenticate` header may identify an HTTP +scheme, but the registry and service-info cannot. A challenge must not trigger +a loop that tries bearer, basic, and local credential stores in turn. +Subsequent commands use the persisted method, so `auto` does not make network +behavior vary from invocation to invocation. `git drs remote diagnose` may +explicitly rerun discovery. + +The common preset path should require no explicit `--auth`. For example, the +CGC and Synapse presets select bearer authentication, while a Terra preset may +select `google-adc`. `--auth` remains available for custom deployments, +overrides, and ambiguous discovery; it is not routine boilerplate. Supplying a +credential source whose type unambiguously implies a method likewise makes an +explicit `--auth` unnecessary. + +`provider-helper` is for authentication lifecycles that the generic methods +cannot represent. Examples include a Gen3 profile helper that exchanges an API +key and refreshes a Fence access token, or a future Synapse helper that obtains +a short-lived token through Synapse's supported login flow. The value does not +mean "execute any program found on `PATH`": the helper must be registered by a +known provider adapter, selected by the detected or explicit `--provider`, and +subject to the endpoint allowlist for that remote. If more than one helper is +available, configuration should use a qualified value such as +`provider-helper:gen3-profile` rather than relying on search order. + +This differs from `--credential helper:synapse`: a **credential helper** only +returns already usable material, such as a bearer token, while a **provider +helper** owns a provider-specific exchange or refresh flow and may use a +credential source as its input. For example: + +```sh +# A generic bearer method reads an already usable token. +git drs remote add synapse https://repo-prod.prod.sagebase.org \ + --auth bearer --credential helper:synapse + +# A provider adapter exchanges the named Gen3 profile and refreshes tokens. +git drs remote add production https://example-gen3.org \ + --provider gen3 --auth provider-helper:gen3-profile \ + --credential profile:production --scope PROGRAM/PROJECT +``` + +### Defaults remove flags + +- Identify the service and provider using GA4GH service-info and a small, + maintained fingerprint table after resolving any explicit registry selector. + Probe resolver and publisher capabilities separately; service-info does not + advertise authentication or upload/registration support. Print the registry + match, endpoint, detected provider, and probed capabilities before persisting + configuration. +- Resolve `--auth auto` using the deterministic authentication selection above. + Do not silently downgrade from authenticated to anonymous access. +- Infer read-only versus writable from capability probing. Replace `--mode` + with reported capabilities rather than a user assertion. +- Default checkout behavior from the repository's existing configuration. + `--checkout` is an optional convenience on `remote add`; the canonical + setting belongs to a repository-level `config` command. +- Prompt for a storage choice only when a write operation actually needs one. + Keep `--storage` for automation, but do not make bucket selection part of + adding a read-only DRS resolver. + +### Credential sources + +One `--credential` option should name where credentials come from: + +```text +env:NAME read a token from an environment variable +file:PATH import/read a provider credential file +helper:NAME invoke an OS, Git, or provider credential helper +stdin read a secret without placing it in shell history +profile:NAME use an existing provider profile +``` + +Inline `--token` and `--password` values should be deprecated because they are +visible in process listings and shell history. Basic authentication can still +be supported by a helper that returns a username and password. Public tracked +configuration contains only endpoint, provider, scope, and safe capability +hints; credential source selection and secrets remain clone-local. + +## Separate the DRS protocol from provider extensions + +Model a remote as a composition instead of one provider-specific client: + +1. **Resolver** — the GA4GH DRS operations for object metadata and access + methods/URLs. +2. **Authenticator** — anonymous, bearer, basic, Google ADC, or a provider + helper that can refresh credentials. +3. **Namespace codec** — parsing and formatting the server's accepted DRS URI + authority and object identifiers. +4. **Publisher** — optional upload, registration, checksum search, bucket + mapping, and delete operations. These are not implied by DRS resolution. +5. **Capabilities** — discovered facts such as resolve, download, search, + register, upload, and delete, cached with the remote configuration and + rechecked by `git drs remote diagnose `. + +The generic resolver should implement the standard DRS object and access +flows. Provider adapters should be small and limited to real differences: +authentication, endpoint/URI normalization, pagination or response quirks, +and non-standard publishing APIs. Core code should select behavior by a +capability interface, not by tests such as `remote.type == "terra"`. + +This separation also makes errors useful. A CGC or Synapse remote can be a +perfectly valid read-only resolver even when it cannot support the current +Gen3 checksum-search and registration workflow. `push` should fail early with +"remote does not advertise register/upload" rather than treating resolution +success as evidence that publishing is available. + +## CGC and Synapse + +Both services should first be treated as GA4GH DRS resolvers, then augmented +only where their documented behavior requires it. + +### Cancer Genomics Cloud + +The CGC adapter should provide a preset for its DRS base URL, bearer-token +credential help, and its accepted DRS URI authority/object identifier form. +It should use the generic object and access flow. Any CGC project browsing, +file upload, or file registration API belongs in an optional publisher adapter, +because those operations are outside the baseline DRS resolver contract. + +The implementation must test whether an access response contains a directly +usable URL or requires the follow-up access endpoint, preserve documented +headers, and never persist a signed access URL. A project identifier, if needed +for non-DRS publishing, is `--scope`; it must not be required merely to resolve +a complete DRS URI. + +### Synapse + +The Synapse adapter should likewise provide its production endpoint/authority +preset and obtain a Synapse bearer credential through a helper. Resolution +should remain object-ID based and should not assume that a returned checksum is +itself a resolvable object ID. + +Synapse project/entity discovery and mutation are separate Synapse REST +capabilities. If later supported for publishing, they should be implemented by +a Synapse publisher and exposed through capability checks, not folded into the +generic DRS resolver or made mandatory during `remote add`. + +### Conformance fixtures + +Add provider contract tests built from sanitized documented responses for: + +- service-info discovery or the explicit fallback when it is unavailable; +- object metadata with one and multiple access methods; +- the access endpoint and expiring URL handling; +- bearer authentication and redaction of tokens and response bodies; +- 401/403, 404, 429, and server-error classification; +- DRS URI authority and percent-encoding normalization; and +- missing SHA-256 checksums (the canonical DRS URI remains the retrieval + identity). + +Live acceptance tests should be opt-in and credential-gated. Generic GA4GH +contract tests should run against every adapter so provider support cannot +silently diverge. + +## Compatibility and migration + +Introduce the unified form without immediately removing scripts: + +1. Add `remote add ` with a derived local name, support + explicit `registry:` lookup, retain the explicit-name form for + overrides, and store the resolved compositional remote configuration. +2. Keep `remote add gen3`, `local`, and `terra` as hidden or deprecated + compatibility shims that translate their arguments to the new model. +3. Add `git drs remote migrate` and make `remote list --verbose` display the + translated endpoint, provider, auth source (never its value), and + capabilities. +4. Warn on inline `--token`/`--password`, `--mode`, and `--no-skip-smudge` with + the exact replacement command for at least one release cycle. +5. Remove legacy forms only in a major release or retain them indefinitely as + thin aliases if maintenance cost is negligible. + +The minimal common case then becomes memorable: + +```sh +git drs remote add +``` + +An explicit local name, scope, authentication override, publishing storage, +and checkout policy appear only when the server or workflow actually needs +them. A credential source may still be required, but a preset or unambiguous +source prevents the user from also having to state the corresponding +authentication method. + +## References + +- [Cancer Genomics Cloud DRS API overview](https://docs.cancergenomicscloud.org/reference/drs-api-overview) +- [GA4GH Service Registry](https://registry.ga4gh.org/) +- [Synapse DRS controller REST documentation](https://rest-docs.synapse.org/rest/index.html#org.sagebionetworks.drs.controller.DrsController) diff --git a/docs/terra-anvil-drs-download-gap-adr.md b/docs/terra-anvil-drs-download-gap-adr.md new file mode 100644 index 00000000..737c4362 --- /dev/null +++ b/docs/terra-anvil-drs-download-gap-adr.md @@ -0,0 +1,359 @@ +# ADR 0004 / GitHub Issue: Support Terra/AnVIL DRS downloads in `git-drs` + +## Status + +Proposed + +## Issue type + +Architecture Decision Record / feature gap analysis + +## Download + +[Download this ADR as Markdown](./terra-anvil-drs-download-gap-adr.md) + +## Related documents + +- [Terra/AnVIL TDD acceptance tests](./terra-anvil-tdd-tests.md) + +## Short answer: Terra DRS service-info endpoint + +Yes. Terra Data Repository exposes the GA4GH DRS service-info endpoint at: + +```text +https://data.terra.bio/ga4gh/drs/v1/service-info +``` + +For `git-drs` Terra health checks, `https://data.terra.bio` is the Terra DRS service base URL and `git drs ping` derives `/ga4gh/drs/v1/service-info` from that base. Terra also uses DRSHub for DRS URI resolution; DRSHub URLs such as `https://drshub.dsde-.broadinstitute.org/api/v4/drs/resolve` are resolver API URLs, not the GA4GH DRS service-info endpoint that `git drs ping` should use. + +## Summary + +`git-drs` already has most of the repository and hydration machinery needed for Terra/AnVIL, so the remaining work should be scoped as an incremental provider/identity extension rather than a wholesale redesign. The most direct path is to add a `terra` remote mode, wire Terra authentication and provider resolution at remote setup time, and then close the two compatibility gaps that are currently not well-defined for Terra references: SHA256-shaped local identity and direct DRS URL pointer identity. A manifest bootstrap workflow remains useful, but it can build on the same lower-level `add-ref`/resolver work instead of being treated as a separate large subsystem. + +The Terra remote endpoint used for service-health checks should be the Terra DRS service base URL, for example `https://data.terra.bio` in production. That base URL supports the GA4GH DRS service-info route at `/ga4gh/drs/v1/service-info`. DRSHub remains relevant for resolving Terra DRS URIs into access information, but it is a resolver service rather than the service-info host for `git drs ping`. + +## Background + +Gregor project users want a Git repository that stores project structure, manifests, analysis code, and `git-drs` pointer files while leaving controlled-access payload bytes on AnVIL. Users should be able to clone normal Git history, review which data objects are referenced, authenticate through AnVIL/Terra access controls, and hydrate only selected paths with commands such as: + +```bash +git drs pull -I "data/cohort-a/**" +``` + +Terra/AnVIL DRS downloads typically start from stable DRS URIs and use Terra-compatible authentication and resolution tooling to obtain authorized access URLs. A repository bootstrap workflow also needs to query AnVIL DRS metadata and generate pointer files and manifests deterministically. + +## Current state + +`git-drs` already provides several useful building blocks: + +- Git-compatible pointer workflows. +- `git drs track` for tracked data path patterns. +- `git drs pull -I ` for selective hydration. +- `git drs add-ref ` as an initial single-object reference workflow. +- Gen3/Syfon remote configuration. +- Local metadata and cache plumbing for DRS-backed objects. + +However, these pieces do not yet compose into a complete Terra/AnVIL workflow. + +## Decision + +Add first-class Terra/AnVIL DRS support as an incremental extension of the +existing remote and reference workflows. The implemented CLI uses the unified +`git drs remote add [name] terra ...` form so authentication and provider +behavior are known at remote configuration time. Once a pointer or reference +is associated with a Terra remote, `git-drs` can use a Terra-aware resolver to +translate the DRS URI into downloadable access URLs. The main unresolved +design decision is how to represent DRS URL identity in pointer files while +preserving the existing SHA256-oriented cache and compatibility assumptions. + +## Goals + +- Allow repositories to store stable Terra/AnVIL DRS references in Git without committing payload bytes. +- Support selective hydration using existing include-filter semantics. +- Support Terra/AnVIL authentication mechanisms such as Google Application Default Credentials or bearer tokens. +- Resolve DRS URIs to authorized access URLs using GA4GH DRS and/or Terra/Martha-compatible resolution. +- Generate deterministic pointer files from manifests or workspace table exports. +- Preserve reviewability of data-reference changes in pull requests. +- Report metadata conflicts, missing checksums, inaccessible DRS IDs, and path collisions clearly. + +## Non-goals + +- Bypass AnVIL authentication or authorization. +- Mirror Gregor payload bytes into Git or unmanaged buckets. +- Replace Terra workspace data management. +- Edit AnVIL source metadata from a Git repository. +- Require all Terra/AnVIL DRS records to have SHA256 checksums. + +## Team feedback incorporated + +This ADR is revised to reflect team feedback that Terra support should be smaller than a broad re-architecture. The preferred shape is: + +- add a `terra` remote mode and wire auth/provider behavior there; +- extend `add-ref` or add a related reference command so callers can create Terra-backed DRS references explicitly; +- resolve Terra references through a remote-aware DRS resolver; +- focus design discussion on SHA256 compatibility and DRS URL pointer compatibility; +- keep manifest/bootstrap support as a batch layer over the reference primitive. + +The ADR also captures an open pointer-format question raised in review: whether `git-drs` should recognize a `git-drs`-specific pointer file whose `oid` is a DRS URI rather than a Git LFS-style SHA256 value, for example: + +```text +version https://calypr.github.io/spec/v1 +oid drs://cgc-ga4gh-api.sbgenomics.com/4c33ae65e4b08832ce3d94e9c +size 11305017366 +``` + +That pointer shape is attractive for reviewability and direct DRS compatibility, but it needs an explicit cache-key strategy because existing `git-drs` code relies on SHA256-shaped OIDs in several places. + +## Gap analysis + +### Gap 1: Missing Terra/AnVIL remote type + +Current remote configuration is centered on Gen3/Syfon remotes with an `organization/project` scope. Team feedback suggests that a `terra` remote mode is the cleanest and smallest enabling feature: auth, provider selection, and resolver behavior can all be wired when the remote is added. + +Example target CLI: + +```bash +git drs remote add anvil terra --checkout hydrate +``` + +The `terra` preset pins the production endpoint, `google-adc` authentication, +and read-only provider behavior. + +### Gap 2: Terra references need a remote-aware `add-ref` path + +Similar reference-building behavior already exists in `add-url`, and `add-ref` already has the right conceptual role for existing DRS objects. The missing piece is a Terra-aware reference flow where the command knows which remote type owns the reference and can call the matching resolver. + +Possible target command shapes: + +```bash +git drs add-ref --remote anvil drs://example/object data/object.cram +git drs add-ref --remote-type terra drs://example/object data/object.cram +``` + +Required behavior: + +1. Resolve the DRS URI using the selected remote. +2. Create a pointer and local metadata without downloading payload bytes. +3. Preserve the DRS URI as canonical remote identity. +4. Store or derive whatever SHA256-compatible local identity `git-drs` needs for cache compatibility. + +### Gap 3: DRS URI pointer compatibility is not well defined + +The existing design still has code paths where local pointer identity is treated as `oid sha256:<64hex>`. Terra/AnVIL can work with that constraint if the DRS URI is stored as first-class metadata and a deterministic SHA256-shaped local cache key is derived from it. However, review feedback also proposes a `git-drs`-specific pointer format where the pointer `oid` itself is a DRS URI. + +Option A: keep Git LFS-shaped pointers and store DRS URI in sidecar/local metadata. + +```text +version https://git-lfs.github.com/spec/v1 +oid sha256: +size +``` + +Option B: add a `git-drs` pointer spec that accepts DRS URI OIDs directly. + +```text +version https://calypr.github.io/spec/v1 +oid drs://example/object +size +``` + +Option A is likely lower-risk because it preserves current SHA256 assumptions. Option B is more reviewable and direct, but requires parser, cache fanout, inventory, and hydration updates so DRS URI OIDs can be normalized into safe local cache keys. + +### Gap 4: SHA256 compatibility is still non-negotiable internally + +Review feedback notes that SHA256 compatibility remains non-negotiable in current `git-drs`. Terra/AnVIL records may still lack SHA256 checksums or provide different checksum types, so the design should preserve a SHA256-shaped local/cache identity even when that identity is derived from the DRS URI rather than from payload content. + +Required behavior: + +- DRS URI is required. +- Size should be captured when available. +- SHA256 should be captured when available. +- Missing checksum should be reported as a warning or policy-controlled condition, not always a hard failure. + +### Gap 5: Missing Terra/Martha-compatible resolver + +Terra downloads often require a resolver layer that can turn a DRS URI into an authorized signed URL. `git-drs` needs a provider abstraction with Terra/AnVIL implementation. + +Suggested interface: + +```go +type DRSResolver interface { + GetObject(ctx context.Context, drsURI string) (*DRSObject, error) + GetAccessURL(ctx context.Context, drsURI string, accessIDOrType string) (*AccessURL, error) +} +``` + +Expected implementations: + +- `syfonResolver` +- `gen3Resolver` +- `ga4ghResolver` +- `terraResolver` + +### Gap 6: Manifest bootstrap/import should be a batch wrapper over reference creation + +`git-drs` still needs a batch utility to initialize or refresh repositories from authoritative AnVIL data sources, but this should be implemented as a thin wrapper around the Terra-aware reference primitive rather than a separate resolver path. + +Target command shape: + +```bash +git drs import-drs-manifest \ + --remote anvil \ + --input manifests/gregor-source.tsv \ + --drs-uri-column drs_uri \ + --path-column repo_path \ + --output-manifest manifests/gregor-drs-manifest.tsv \ + --data-root data +``` + +Required behavior: + +- Read TSV/CSV manifests or workspace table exports. +- Resolve each DRS URI. +- Validate metadata. +- Write deterministic pointer files. +- Write or update local `git-drs` metadata. +- Write a deterministic output manifest. +- Support dry-run mode. +- Report additions, changes, deletions, collisions, inaccessible IDs, and metadata conflicts. + +### Gap 7: Missing steward/reviewer manifest output + +Bootstrap should create a stable, reviewable manifest recording at least: + +- repository path; +- DRS URI; +- DRS ID; +- object name; +- size; +- checksum type and value when available; +- access method types; +- source workspace/table/row metadata when available; +- metadata status. + +Run-specific details such as timestamps should either be omitted from the committed manifest or isolated in an optional report artifact to preserve idempotency. + +### Gap 8: Current scope model does not match Terra discovery + +Terra/AnVIL source metadata may come from workspace tables, Data Repository Service snapshots, manifests, or other project-specific exports. The canonical scope for hydration should be the DRS URI stored in metadata, not a Gen3-style `organization/project` plus bucket mapping. + +### Gap 9: Access method selection needs policy + +Terra/AnVIL DRS objects may expose multiple access methods. `git-drs` should support explicit and default access preferences. + +Example target flags: + +```bash +git drs pull --access-method auto +git drs pull --access-method https +git drs pull --access-method gs +``` + +### Gap 10: Authentication UX is not Terra-native + +Terra users should be able to configure auth using expected local mechanisms. + +Target options: + +```bash +--auth google-adc +--auth gcloud +--auth bearer-token +``` + +Error messages should distinguish missing credentials, expired credentials, 401 unauthorized, 403 forbidden, inaccessible controlled-access records, and expired signed URLs. + +### Gap 11: Need read-only remote mode + +Gregor/AnVIL repositories reference existing controlled-access data. They should not accidentally trigger payload upload or DRS registration. + +Target behavior: + +```bash +git drs remote add anvil terra --checkout hydrate +``` + +In read-only mode: + +- `git drs pull` works. +- manifest import/bootstrap works. +- upload/register push behavior is disabled or fails with a clear message. + +### Gap 12: Missing Terra/AnVIL documentation + +Add documentation covering: + +- required credentials; +- remote setup; +- manifest input format; +- bootstrap workflow; +- refresh workflow; +- selective hydration; +- missing checksum policy; +- controlled-access troubleshooting; +- common 401/403 causes. + +## Proposed implementation plan + +### Phase 1: Terra remote and resolver + +- Add Terra support to the unified `git drs remote add [name] terra ...` path. +- Wire Terra auth/provider configuration at remote setup time. +- Add a Terra-aware resolver used by reference creation and hydration. +- Add provider-specific diagnostics for authorization failures. + +### Phase 2: Reference creation and pointer identity + +- Extend `add-ref` or add a related command to create Terra-backed references. +- Persist the DRS URI as canonical remote identity. +- Preserve a SHA256-shaped internal/local cache identity, either from a real checksum or a deterministic DRS URI-derived key. +- Decide whether to support direct `oid drs://...` `git-drs` pointer files in addition to Git LFS-shaped pointers. + +### Phase 3: Manifest bootstrap/import + +- Add `git drs import-drs-manifest` or equivalent as a batch wrapper over Terra-aware reference creation. +- Generate pointer files without downloading payload bytes. +- Generate deterministic output manifests. +- Add dry-run, conflict reporting, and deletion policy options. + +### Phase 4: Documentation and acceptance tests + +- Add `docs/terra-anvil.md`. +- Add unit tests for DRS-URI-derived local OIDs. +- Add resolver tests for mocked GA4GH/Terra DRS responses. +- Add integration-style tests for manifest import and selective hydration. + +## Acceptance criteria + +- A maintainer can configure a Terra/AnVIL read-only remote without Gen3/Syfon bucket mapping. +- A bootstrap/import command can generate pointer files from a representative AnVIL manifest without downloading payload bytes. +- Generated pointer metadata includes canonical DRS URIs. +- Missing SHA256 checksums are handled according to documented policy. +- Rerunning bootstrap with unchanged AnVIL metadata produces no Git diff. +- `git drs pull -I ` hydrates selected Terra/AnVIL pointer files after authentication. +- 401/403 and inaccessible DRS records produce actionable error messages. +- Documentation explains credentials, bootstrap inputs, outputs, refresh workflow, and troubleshooting. + +## Open questions + +- The canonical command is the unified `git drs remote add [name] terra` + preset form; the earlier provider-subcommand alternatives are superseded. +- Which AnVIL/Terra endpoint should be treated as canonical for Gregor DRS discovery? +- Should Martha/Terra resolution be implemented directly or invoked through Terra Notebook Utils behavior? +- What should the default policy be for records that lack checksums? +- Should deleted or withdrawn AnVIL records remove pointers automatically, create tombstones, or require explicit maintainer action? +- Should bootstrap/import live in core `git-drs` or as a project-local Gregor script first? + +## Priority + +High for Terra/AnVIL adoption. + +## Labels + +- `adr` +- `enhancement` +- `terra` +- `anvil` +- `drs` +- `bootstrap` +- `controlled-access` diff --git a/docs/terra-anvil-tdd-tests.md b/docs/terra-anvil-tdd-tests.md new file mode 100644 index 00000000..eb52ec7a --- /dev/null +++ b/docs/terra-anvil-tdd-tests.md @@ -0,0 +1,198 @@ +# Terra/AnVIL TDD acceptance tests + +## Purpose + +This document explains the test-driven development approach for the Terra/AnVIL DRS work and how to run the tests that currently describe the desired behavior. + +The tests are intentionally written before the production implementation. They are expected to fail until `git-drs` supports: + +- a `terra` remote type; +- Terra-aware DRS reference creation; +- DRS URI-shaped `git-drs` pointer files; +- deterministic SHA256-shaped local cache identity for DRS URI references. + +These tests turn the ADR acceptance criteria into executable checks so implementation can proceed incrementally. + +## Test files + +### `internal/config/terra_remote_acceptance_test.go` + +Covers the Terra remote configuration acceptance criterion. + +The test writes Git config entries equivalent to a future Terra remote: + +```ini +[drs] + default-remote = anvil +[drs "remote.anvil"] + type = terra + endpoint = https://data.terra.bio + auth = google-adc + mode = read-only +``` + +Expected future behavior: + +- `LoadConfig` recognizes `type = terra`. +- `Config.GetRemote("anvil")` returns a non-nil DRS remote. +- The remote preserves the configured Terra DRS endpoint. + +Current behavior: + +- The test fails because config parsing only models existing Gen3/local remotes. + +### `cmd/addref/terra_addref_acceptance_test.go` + +Covers the remote-aware reference creation acceptance criterion. + +Expected future behavior: + +- `git drs add-ref` exposes a `--remote-type` flag or equivalent resolver-selection mechanism. +- Terra DRS references can select Terra resolver behavior explicitly when creating pointer metadata. + +Current behavior: + +- The test fails because `add-ref` does not expose `--remote-type`. + +### `cmd/ping/main_test.go` + +Covers the Terra remote connectivity acceptance criterion for `git drs ping`. + +The acceptance test configures a future Terra remote named `anvil` and uses an +`httptest` server to stand in for the Terra/AnVIL DRS endpoint: + +```ini +[drs] + default-remote = anvil +[drs "remote.anvil"] + type = terra + endpoint = http://127.0.0.1: + auth = google-adc + mode = read-only +``` + +Expected future behavior: + +- `git drs ping anvil` recognizes the configured `terra` remote. +- The command prints Terra remote status, including `type: terra` and the + configured endpoint. +- The command pings the Terra/AnVIL DRS service-info route at + `/ga4gh/drs/v1/service-info`. +- A successful service-info response is reported as `health: ok`. + +Current behavior: + +- The test passes once runtime construction recognizes Terra remotes and + `git drs ping` checks the configured DRS service-info endpoint. +- This test should remain a regression test for Terra/AnVIL ping connectivity. + +### `internal/lfs/terra_pointer_acceptance_test.go` + +Covers the DRS URI pointer compatibility and cache identity acceptance criteria. + +Expected future pointer behavior: + +```text +version https://calypr.github.io/spec/v1 +oid drs://cgc-ga4gh-api.sbgenomics.com/4c33ae65e4b08832ce3d94e9c +size 11305017366 +``` + +Expected future cache behavior: + +- A DRS URI pointer can still be mapped to a deterministic SHA256-shaped local cache key. +- The cache key may be derived from a normalized DRS URI rather than from payload content. +- Existing cache fanout assumptions can remain SHA256-shaped even when the pointer `oid` is a DRS URI. + +Current behavior: + +- The pointer parser rejects non-`sha256` OID types. +- `ObjectPath` rejects raw DRS URIs because it currently requires a 64-character SHA256 hex string. + +## How to run the focused tests + +From the repository root, run: + +```bash +go test -timeout 30s ./internal/lfs ./internal/config ./cmd/addref ./cmd/ping +``` + +These tests are currently expected to fail. A failure is useful because it confirms the tests are exercising missing Terra/AnVIL behavior rather than silently passing against existing Gen3/Syfon-only behavior. + +## Expected current output shape + +The exact timing may differ, but the failures should include messages like: + +```text +--- FAIL: TestAcceptanceTerraDRSURIoidPointerIsRecognized + terra_pointer_acceptance_test.go: expected git-drs DRS URI pointer to be recognized + +--- FAIL: TestAcceptanceTerraDRSURIoidPointerHasDeterministicSHA256CacheKey + terra_pointer_acceptance_test.go: expected DRS URI to be normalized to a sha256-shaped cache key path + +--- FAIL: TestAcceptanceLoadTerraRemoteConfig + terra_remote_acceptance_test.go: expected terra remote to load as a DRS remote + +--- FAIL: TestAcceptanceAddRefExposesRemoteTypeForTerraReferences + terra_addref_acceptance_test.go: expected add-ref to expose --remote-type + +``` + +## How to run individual test groups + +### Pointer and cache identity tests + +```bash +go test -timeout 30s ./internal/lfs -run 'TestAcceptanceTerraDRSURIoidPointer' +``` + +Use this while implementing pointer parsing, DRS URI normalization, and cache-key derivation. + +### Terra remote config test + +```bash +go test -timeout 30s ./internal/config -run TestAcceptanceLoadTerraRemoteConfig +``` + +Use this while adding config structs, parsing, and persistence for `type = terra` remotes. + +### Terra-aware `add-ref` test + +```bash +go test -timeout 30s ./cmd/addref -run TestAcceptanceAddRefExposesRemoteTypeForTerraReferences +``` + +Use this while adding CLI surface for remote-aware Terra reference creation. + +### Terra-aware `ping` test + +```bash +go test -timeout 30s ./cmd/ping -run TestAcceptancePingTerraDRSServer +``` + +Use this while wiring Terra remotes into runtime construction and adding the +Terra/AnVIL DRS service-info health probe used by `git drs ping`. + +## Recommended implementation order + +1. Add a Terra remote model to config parsing and persistence. +2. Add a Terra-aware resolver abstraction that can resolve DRS URIs through the configured remote. +3. Add a Terra-aware ping path that checks the configured DRS service-info endpoint. +4. Extend `add-ref` so it can create references using a selected remote or remote type. +5. Decide the pointer-format strategy: + - keep Git LFS-shaped `oid sha256:` pointers with DRS URI in metadata; or + - support `version https://calypr.github.io/spec/v1` with `oid drs://...` directly. +6. Add deterministic DRS URI to SHA256-shaped cache-key derivation. +7. Add manifest bootstrap/import as a batch wrapper over the reference creation primitive. + +## When the tests should pass + +The focused tests should pass when `git-drs` can: + +- load a read-only Terra remote from Git config; +- ping the configured Terra/AnVIL DRS service-info endpoint; +- expose a Terra resolver selection path for `add-ref`; +- parse DRS URI-shaped `git-drs` pointer files; +- map DRS URI identities to deterministic SHA256-shaped cache paths. + +At that point, the failing TDD tests should be treated as regression tests for the first Terra/AnVIL support increment. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index a65142f7..f951d23b 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -72,7 +72,7 @@ Those changes persist in the clone. They are not something you redo per session. Examples: -- `git drs remote add gen3 ...` +- `git drs remote add [name] ...` - `git drs remote remove ...` - `git drs init` - `git drs track` @@ -188,6 +188,11 @@ git drs pull ### `git drs ls-files` does not show my file +This is not expected for a pointer created by a current `git drs add-ref`: +`add-ref` automatically adds its destination to `.gitattributes`. For pointers +created with an older version, add the tracking rule with +`git drs track path/to/file` and stage `.gitattributes`. + Check these in order: 1. is the path actually tracked? @@ -214,6 +219,22 @@ git ls-files -- path/to/file git drs ls-files -l ``` +### `git add ... git drs add-ref --remote ...` reports `unknown option 'remote'` + +`git add` and `git drs add-ref` are separate commands. If they are entered on +the same command line, Git interprets `--remote` as an option to `git add`, +which does not have that option. + +Create the reference first, then stage the generated pointer and tracking rule: + +```bash +git drs add-ref --remote anvil drs://drs.anv0:v2_example subject.tsv +git add .gitattributes subject.tsv +git commit -m "Add subject.tsv reference" +``` + +Do not prefix the `git drs add-ref` command with `git add`. + ### `git remote remove` did not remove my `git-drs` remote That is expected. @@ -271,14 +292,18 @@ If needed, inspect the detailed logs: ls -la .git/drs/ ``` -### `git drs remote add gen3` fails on bucket mapping +### `git drs remote add` fails on bucket mapping Current shape: ```bash -git drs remote add gen3 [remote-name] [--cred | --token ] +git drs remote add [remote-name] calypr --scope \ + --credential ``` +Credential sources use an explicit scheme such as `file:/path/to/key.json`, +`env:GEN3_TOKEN`, or `profile:production`; inline secrets are not accepted. + If this fails, the likely cause is missing bucket mapping for that scope. That mapping is usually steward/admin setup, not something the end user invents ad hoc. @@ -288,7 +313,8 @@ That mapping is usually steward/admin setup, not something the end user invents Refresh by re-adding the remote with a new credential file or token: ```bash -git drs remote add gen3 production HTAN_INT/BForePC --cred /path/to/new-credentials.json +git drs remote add production calypr --scope HTAN_INT/BForePC \ + --credential file:/path/to/new-credentials.json ``` You do not need to run `git drs init` again. @@ -296,14 +322,16 @@ You do not need to run `git drs init` again. What `git-drs` does automatically: - if the stored access token is expired but the stored API key is still valid, `git-drs` will attempt to refresh the access token -- if the API key itself is expired, revoked, or replaced, you need to re-run `git drs remote add gen3 ...` +- if the API key itself is expired, revoked, or replaced, re-run the unified + `git drs remote add ... --credential ` command How to think about recovery: - token expired, key still valid: - often automatic - key expired or replaced: - - rerun `git drs remote add gen3 ... --cred ...` or `--token ...` + - rerun `git drs remote add ... --credential file:` or use an + environment credential source such as `--credential env:GEN3_TOKEN` How to check what is in use: @@ -315,7 +343,9 @@ And for the underlying Gen3 profile data: - inspect `~/.gen3/gen3_client_config.ini` -If you want the least surprising fix, just re-run `git drs remote add gen3 ...` with the current credential file. That updates the stored profile and repo token plumbing in one step. +If you want the least surprising fix, re-run the unified `git drs remote add` +command with the current `--credential file:`. That updates the stored +profile and repo token plumbing in one step. ### `git drs push` fails with upload or register errors @@ -392,7 +422,8 @@ git drs pull -I "*.bam" If you want Git DRS to always download and hydrate all file payloads automatically during checkouts (reverting to non-skip behavior), you can: - Configure it during remote setup: ```bash - git drs remote add gen3 public HTAN_INT/BForePC --no-skip-smudge + git drs remote add public calypr --scope HTAN_INT/BForePC \ + --credential file:~/.gen3/credentials.json --checkout hydrate ``` - Or configure it directly in Git settings: ```bash diff --git a/internal/config/config.go b/internal/config/config.go index 6006626e..24a95a27 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,8 +3,9 @@ package config import ( "errors" "fmt" - "path/filepath" + "os/exec" "sort" + "strconv" "strings" "github.com/calypr/git-drs/internal/gitrepo" @@ -20,6 +21,8 @@ const ( Gen3ServerType RemoteType = "gen3" LocalServerType RemoteType = "local" + TerraServerType RemoteType = "terra" + GA4GHServerType RemoteType = "ga4gh" configSection = "drs" remoteSubsectionPrefix = "remote." @@ -42,6 +45,10 @@ func (c Config) GetRemote(remote Remote) DRSRemote { return x.Gen3 } else if x.Local != nil { return x.Local + } else if x.Terra != nil { + return x.Terra + } else if x.Generic != nil { + return x.Generic } return nil } @@ -94,10 +101,6 @@ func getRepo() (*git.Repository, error) { return gitrepo.GetRepo() } -func (c Config) ConfigPath() (string, error) { - return getConfigPath() -} - // updates and git adds a Git DRS config file // this should handle three cases: // 1. create a new config file if it does not exist / is empty @@ -130,6 +133,15 @@ func UpdateRemote(name Remote, remote RemoteSelect) (*Config, error) { if remote.Gen3.StoragePrefix != "" { remoteSubsection.SetOption("storage_prefix", remote.Gen3.StoragePrefix) } + } else if remote.Terra != nil { + remoteSubsection.SetOption("type", "terra") + remoteSubsection.SetOption("endpoint", remote.Terra.Endpoint) + if remote.Terra.Auth != "" { + remoteSubsection.SetOption("auth", remote.Terra.Auth) + } + if remote.Terra.Mode != "" { + remoteSubsection.SetOption("mode", remote.Terra.Mode) + } } else if remote.Local != nil { remoteSubsection.SetOption("type", "local") remoteSubsection.SetOption("endpoint", remote.Local.BaseURL) @@ -145,6 +157,20 @@ func UpdateRemote(name Remote, remote RemoteSelect) (*Config, error) { if remote.Local.StoragePrefix != "" { remoteSubsection.SetOption("storage_prefix", remote.Local.StoragePrefix) } + } else if remote.Generic != nil { + r := remote.Generic + remoteSubsection.SetOption("type", "ga4gh") + remoteSubsection.SetOption("endpoint", r.Endpoint) + remoteSubsection.SetOption("provider", r.Provider) + remoteSubsection.SetOption("auth", r.Auth) + for key, value := range map[string]string{"credential": r.Credential, "scope": r.Scope, "storage": r.Storage, "checkout": r.Checkout, "preset": r.Preset, "registry-service-id": r.RegistryServiceID} { + if value != "" { + remoteSubsection.SetOption(key, value) + } + } + if r.PresetVersion > 0 { + remoteSubsection.SetOption("preset-version", fmt.Sprint(r.PresetVersion)) + } } // Set default remote if not set @@ -162,7 +188,7 @@ func UpdateRemote(name Remote, remote RemoteSelect) (*Config, error) { return LoadConfig() } -func parseAndAddRemote(cfg *Config, subsectionName string, remoteType string, endpoint string, project string, bucket string, organization string, storagePrefix string) { +func parseAndAddRemote(cfg *Config, subsectionName string, remoteType string, endpoint string, project string, bucket string, organization string, storagePrefix string, auth string, mode string) { if !strings.HasPrefix(subsectionName, remoteSubsectionPrefix) { return } @@ -178,6 +204,12 @@ func parseAndAddRemote(cfg *Config, subsectionName string, remoteType string, en Organization: organization, StoragePrefix: storagePrefix, } + } else if remoteType == "terra" { + rs.Terra = &TerraRemote{ + Endpoint: endpoint, + Auth: auth, + Mode: mode, + } } else if remoteType == "local" { rs.Local = &LocalRemote{ BaseURL: endpoint, @@ -191,6 +223,78 @@ func parseAndAddRemote(cfg *Config, subsectionName string, remoteType string, en cfg.Remotes[remoteName] = rs } +func addGenericRemote(cfg *Config, name Remote, opts map[string]string) { + version, _ := strconv.Atoi(opts["preset-version"]) + cfg.Remotes[name] = RemoteSelect{Generic: &GenericRemote{ + Endpoint: opts["endpoint"], Provider: opts["provider"], Auth: opts["auth"], + Credential: opts["credential"], Scope: opts["scope"], Storage: opts["storage"], + Checkout: opts["checkout"], Preset: opts["preset"], PresetVersion: version, + RegistryServiceID: opts["registry-service-id"], + }} +} + +func loadGitConfigOverrides(cfg *Config) error { + cmd := exec.Command("git", "config", "--local", "--get-regexp", `^drs\.`) + out, err := cmd.Output() + if err != nil { + // git config exits non-zero when no matching keys exist. In that case, + // the go-git result above is still the complete repository-local config. + return nil + } + + remoteOptions := make(map[Remote]map[string]string) + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + key, value, ok := strings.Cut(line, " ") + if !ok { + continue + } + value = strings.TrimSpace(value) + if key == "drs.default-remote" { + cfg.DefaultRemote = Remote(value) + continue + } + const prefix = "drs.remote." + if !strings.HasPrefix(key, prefix) { + continue + } + rest := strings.TrimPrefix(key, prefix) + idx := strings.LastIndex(rest, ".") + if idx <= 0 || idx == len(rest)-1 { + continue + } + name := Remote(rest[:idx]) + option := rest[idx+1:] + if remoteOptions[name] == nil { + remoteOptions[name] = make(map[string]string) + } + remoteOptions[name][option] = value + } + + for name, opts := range remoteOptions { + if opts["type"] == "ga4gh" { + addGenericRemote(cfg, name, opts) + continue + } + parseAndAddRemote( + cfg, + remoteSubsectionPrefix+string(name), + opts["type"], + opts["endpoint"], + opts["project"], + opts["bucket"], + opts["organization"], + opts["storage_prefix"], + opts["auth"], + opts["mode"], + ) + } + return nil +} + // LoadConfig loads configuration using go-git func LoadConfig() (*Config, error) { repo, err := getRepo() @@ -206,7 +310,6 @@ func LoadConfig() (*Config, error) { cfg := &Config{ Remotes: make(map[Remote]RemoteSelect), } - // Iterate over all sections to find 'drs' and its subsections for _, section := range conf.Raw.Sections { if section.Name != configSection { @@ -223,6 +326,14 @@ func LoadConfig() (*Config, error) { if !strings.HasPrefix(subsection.Name, remoteSubsectionPrefix) { continue } + if subsection.Option("type") == "ga4gh" { + opts := make(map[string]string) + for _, key := range []string{"endpoint", "provider", "auth", "credential", "scope", "storage", "checkout", "preset", "preset-version", "registry-service-id"} { + opts[key] = subsection.Option(key) + } + addGenericRemote(cfg, Remote(strings.TrimPrefix(subsection.Name, remoteSubsectionPrefix)), opts) + continue + } parseAndAddRemote( cfg, subsection.Name, @@ -232,10 +343,16 @@ func LoadConfig() (*Config, error) { subsection.Option("bucket"), subsection.Option("organization"), subsection.Option("storage_prefix"), + subsection.Option("auth"), + subsection.Option("mode"), ) } } + if err := loadGitConfigOverrides(cfg); err != nil { + return nil, err + } + return cfg, nil } @@ -285,6 +402,16 @@ func RemoveRemote(name Remote) (*Config, error) { fmt.Sprintf("drs.remote.%s.bucket", name), fmt.Sprintf("drs.remote.%s.organization", name), fmt.Sprintf("drs.remote.%s.storage_prefix", name), + fmt.Sprintf("drs.remote.%s.auth", name), + fmt.Sprintf("drs.remote.%s.mode", name), + fmt.Sprintf("drs.remote.%s.provider", name), + fmt.Sprintf("drs.remote.%s.credential", name), + fmt.Sprintf("drs.remote.%s.scope", name), + fmt.Sprintf("drs.remote.%s.storage", name), + fmt.Sprintf("drs.remote.%s.checkout", name), + fmt.Sprintf("drs.remote.%s.preset", name), + fmt.Sprintf("drs.remote.%s.preset-version", name), + fmt.Sprintf("drs.remote.%s.registry-service-id", name), fmt.Sprintf("drs.remote.%s.token", name), fmt.Sprintf("drs.remote.%s.username", name), fmt.Sprintf("drs.remote.%s.password", name), @@ -323,15 +450,3 @@ func firstRemote(cfg *Config) Remote { sort.Strings(names) return Remote(names[0]) } - -// GetGitConfigInt reads an integer value from git config -// getGitConfigValue retrieves a value from git config by key -func getConfigPath() (string, error) { - topLevel, err := gitrepo.GitTopLevel() - if err != nil { - return "", err - } - - configPath := filepath.Join(topLevel, gitrepo.DRSDir, gitrepo.ConfigYAML) - return configPath, nil -} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7b6ab109..739135f5 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -3,9 +3,27 @@ package config import ( "os" "os/exec" + "path/filepath" "testing" ) +func TestLoadConfigIgnoresFormerRepositoryYAMLPath(t *testing.T) { + dir := setupTestRepo(t) + if err := os.MkdirAll(filepath.Join(dir, ".git-drs"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ".git-drs", "config.yaml"), []byte("not: [valid"), 0o644); err != nil { + t.Fatal(err) + } + cfg, err := LoadConfig() + if err != nil { + t.Fatalf("former repository YAML path must not be loaded: %v", err) + } + if len(cfg.Remotes) != 0 { + t.Fatalf("unexpected remotes loaded from former YAML path: %+v", cfg.Remotes) + } +} + func setupTestRepo(t *testing.T) string { t.Helper() @@ -291,3 +309,38 @@ func TestUpdateRemote_LocalTypePersistence(t *testing.T) { t.Errorf("Expected BaseURL http://localhost:8080, got %s", localRemote.BaseURL) } } + +func TestRemoveRemote_TerraCleansAuthAndMode(t *testing.T) { + tmpDir := setupTestRepo(t) + remoteName := Remote("anvil") + + _, err := UpdateRemote(remoteName, RemoteSelect{ + Terra: &TerraRemote{ + Endpoint: "https://data.terra.bio", + Auth: "google-adc", + Mode: "read-only", + }, + }) + if err != nil { + t.Fatalf("UpdateRemote failed: %v", err) + } + + cfg, err := RemoveRemote(remoteName) + if err != nil { + t.Fatalf("RemoveRemote failed: %v", err) + } + if _, ok := cfg.Remotes[remoteName]; ok { + t.Fatalf("removed Terra remote %q was recreated", remoteName) + } + + for _, key := range []string{ + "drs.remote.anvil.auth", + "drs.remote.anvil.mode", + } { + cmd := exec.Command("git", "config", "--local", "--get", key) + cmd.Dir = tmpDir + if output, err := cmd.CombinedOutput(); err == nil { + t.Errorf("expected %s to be unset, got %q", key, string(output)) + } + } +} diff --git a/internal/config/remote.go b/internal/config/remote.go index f08ae114..a7b5fd40 100644 --- a/internal/config/remote.go +++ b/internal/config/remote.go @@ -1,5 +1,7 @@ package config +import "strings" + type DRSRemote interface { GetProjectId() string GetOrganization() string @@ -9,10 +11,26 @@ type DRSRemote interface { } type RemoteSelect struct { - Gen3 *Gen3Remote - Local *LocalRemote + Gen3 *Gen3Remote + Local *LocalRemote + Terra *TerraRemote + Generic *GenericRemote +} + +// GenericRemote is the compositional configuration produced by the unified +// remote-add command. Credential contains a source identifier, never a secret. +type GenericRemote struct { + Endpoint, Provider, Auth, Credential, Scope, Storage, Checkout string + Preset, RegistryServiceID string + PresetVersion int } +func (r GenericRemote) GetProjectId() string { _, p, _ := strings.Cut(r.Scope, "/"); return p } +func (r GenericRemote) GetOrganization() string { o, _, _ := strings.Cut(r.Scope, "/"); return o } +func (r GenericRemote) GetEndpoint() string { return r.Endpoint } +func (r GenericRemote) GetBucketName() string { b, _, _ := strings.Cut(r.Storage, "/"); return b } +func (r GenericRemote) GetStoragePrefix() string { _, p, _ := strings.Cut(r.Storage, "/"); return p } + type Gen3Remote struct { Endpoint string `yaml:"endpoint"` ProjectID string `yaml:"project_id"` @@ -27,6 +45,18 @@ func (s Gen3Remote) GetEndpoint() string { return s.Endpoint } func (s Gen3Remote) GetBucketName() string { return s.Bucket } func (s Gen3Remote) GetStoragePrefix() string { return s.StoragePrefix } +type TerraRemote struct { + Endpoint string `yaml:"endpoint"` + Auth string `yaml:"auth"` + Mode string `yaml:"mode"` +} + +func (t TerraRemote) GetProjectId() string { return "" } +func (t TerraRemote) GetOrganization() string { return "" } +func (t TerraRemote) GetEndpoint() string { return t.Endpoint } +func (t TerraRemote) GetBucketName() string { return "" } +func (t TerraRemote) GetStoragePrefix() string { return "" } + type LocalRemote struct { BaseURL string ProjectID string diff --git a/internal/filter/clean.go b/internal/filter/clean.go index bb0b019d..5d7d4004 100644 --- a/internal/filter/clean.go +++ b/internal/filter/clean.go @@ -8,7 +8,10 @@ import ( "io" "log/slog" "os" + "os/exec" "path/filepath" + "strconv" + "strings" "github.com/calypr/git-drs/internal/drsobject" "github.com/calypr/git-drs/internal/gitrepo" @@ -67,14 +70,31 @@ func CleanContent(_ context.Context, lfsRoot, pathname string, content io.Reader size := written oid := hex.EncodeToString(h.Sum(nil)) + // A DRS pointer can use the durable DRS URI as its oid rather than the + // payload checksum. When Git asks the clean filter to inspect a hydrated + // file, preserve that indexed pointer if its recorded checksum proves the + // worktree payload is unchanged. Otherwise hydration would turn the URI + // pointer into a SHA256 LFS pointer and appear as a staged modification. + if pointer, ok := matchingIndexedDRSPointer(pathname, oid, size); ok { + if _, err := dst.Write(pointer); err != nil { + return fmt.Errorf("clean: write indexed DRS pointer: %w", err) + } + logger.Debug("clean: restored indexed DRS pointer for hydrated content", "pathname", pathname, "oid", oid, "size", size) + return nil + } + if size > 0 && size < 2048 { if data, readErr := os.ReadFile(tmpPath); readErr == nil { if pointerOID, pointerSize, ok := lfs.ParseLFSPointer(data); ok { if _, err := dst.Write(data); err != nil { return fmt.Errorf("clean: write existing pointer: %w", err) } - if mapErr := writeDrsMap(pathname, pointerOID, pointerSize); mapErr != nil { - logger.Warn("clean: failed to write DRS map entry for existing pointer", "pathname", pathname, "error", mapErr) + // DRS URI pointers already carry their durable lookup identity. The + // SHA256-keyed sidecar map is only applicable to SHA256 pointers. + if !lfs.IsDRSURI(pointerOID) { + if mapErr := writeDrsMap(pathname, pointerOID, pointerSize); mapErr != nil { + logger.Warn("clean: failed to write DRS map entry for existing pointer", "pathname", pathname, "error", mapErr) + } } logger.Debug("clean: passed through existing LFS pointer", "pathname", pathname, "oid", pointerOID, "size", pointerSize) return nil @@ -109,3 +129,60 @@ func CleanContent(_ context.Context, lfsRoot, pathname string, content io.Reader return nil } + +func matchingIndexedDRSPointer(pathname, contentOID string, size int64) ([]byte, bool) { + pathname = filepath.ToSlash(filepath.Clean(pathname)) + if pathname == "." || pathname == ".." || strings.HasPrefix(pathname, "../") { + return nil, false + } + cmd := exec.Command("git", "show", ":"+pathname) + pointer, err := cmd.Output() + if err != nil { + return nil, false + } + pointerOID, pointerSize, ok := lfs.ParseLFSPointer(pointer) + if !ok || !lfs.IsDRSURI(pointerOID) || pointerSize != size { + return nil, false + } + // Some DRS services do not publish a SHA256 checksum. In that case compare + // the worktree payload with the validated object cached by pull. This keeps + // status clean without hiding a same-sized edit made after hydration. + if cachedOID, ok := cachedObjectOID(pointerOID, size); ok && cachedOID == contentOID { + return pointer, true + } + for _, line := range strings.Split(string(pointer), "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && strings.EqualFold(fields[0], "sha256") { + checksum := strings.TrimPrefix(strings.ToLower(fields[1]), "sha256:") + if checksum == contentOID { + return pointer, true + } + } + if len(fields) == 2 && fields[0] == "size" { + // Reject malformed sizes even if a future pointer parser becomes + // more permissive. + if parsed, err := strconv.ParseInt(fields[1], 10, 64); err != nil || parsed != size { + return nil, false + } + } + } + return nil, false +} + +func cachedObjectOID(pointerOID string, size int64) (string, bool) { + cachePath, err := lfs.ObjectPath(gitrepo.LFSObjectsPath, pointerOID) + if err != nil { + return "", false + } + f, err := os.Open(cachePath) + if err != nil { + return "", false + } + defer f.Close() + h := sha256.New() + written, err := io.Copy(h, f) + if err != nil || written != size { + return "", false + } + return hex.EncodeToString(h.Sum(nil)), true +} diff --git a/internal/filter/clean_test.go b/internal/filter/clean_test.go index fbfd1766..af7901d1 100644 --- a/internal/filter/clean_test.go +++ b/internal/filter/clean_test.go @@ -5,10 +5,13 @@ import ( "context" "crypto/sha256" "encoding/hex" + "fmt" "io" "log/slog" "os" + "os/exec" "path/filepath" + "strings" "testing" "github.com/calypr/git-drs/internal/drsobject" @@ -69,3 +72,86 @@ func TestCleanContentPassesThroughExistingPointer(t *testing.T) { } } } + +func TestCleanContentPreservesIndexedDRSPointerForHydratedPayload(t *testing.T) { + repo := t.TempDir() + payload := []byte("hydrated population data\n") + pointer := "version https://calypr.github.io/spec/v1\n" + + "oid drs://drs.anv0:v2_example\n" + + "size " + fmt.Sprint(len(payload)) + "\n" + path := filepath.Join(repo, "population_descriptor.tsv") + if err := os.WriteFile(path, []byte(pointer), 0o644); err != nil { + t.Fatalf("write pointer: %v", err) + } + for _, args := range [][]string{{"init"}, {"add", "population_descriptor.tsv"}} { + cmd := exec.Command("git", args...) + cmd.Dir = repo + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + } + + orig, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + defer os.Chdir(orig) + if err := os.Chdir(repo); err != nil { + t.Fatalf("chdir: %v", err) + } + cachePath, err := lfs.ObjectPath(gitrepo.LFSObjectsPath, "drs://drs.anv0:v2_example") + if err != nil { + t.Fatalf("ObjectPath: %v", err) + } + if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { + t.Fatalf("mkdir cache: %v", err) + } + if err := os.WriteFile(cachePath, payload, 0o644); err != nil { + t.Fatalf("write cache: %v", err) + } + var out bytes.Buffer + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + if err := CleanContent(context.Background(), filepath.Join(repo, ".git", "lfs"), "population_descriptor.tsv", bytes.NewReader(payload), &out, logger); err != nil { + t.Fatalf("CleanContent: %v", err) + } + if out.String() != pointer { + t.Fatalf("expected indexed DRS pointer, got %q", out.String()) + } + + changed := bytes.Repeat([]byte("x"), len(payload)) + out.Reset() + if err := CleanContent(context.Background(), filepath.Join(repo, ".git", "lfs"), "population_descriptor.tsv", bytes.NewReader(changed), &out, logger); err != nil { + t.Fatalf("CleanContent changed payload: %v", err) + } + if out.String() == pointer { + t.Fatal("same-sized changed payload must not be hidden by the indexed DRS pointer") + } +} + +func TestCleanContentPassesThroughDRSURIWithoutSHA256MapWarning(t *testing.T) { + repo := t.TempDir() + orig, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + defer os.Chdir(orig) + if err := os.Chdir(repo); err != nil { + t.Fatalf("chdir: %v", err) + } + + const pointer = "version https://calypr.github.io/spec/v1\n" + + "oid drs://drs.anv0:v2_e68887be-c583-375a-a773-48771192c8fa\n" + + "size 200184\n" + var out bytes.Buffer + var logs bytes.Buffer + logger := slog.New(slog.NewTextHandler(&logs, nil)) + if err := CleanContent(context.Background(), filepath.Join(repo, ".git", "lfs"), "population_descriptor.tsv", bytes.NewBufferString(pointer), &out, logger); err != nil { + t.Fatalf("CleanContent returned error: %v", err) + } + if out.String() != pointer { + t.Fatalf("expected DRS pointer passthrough, got %q", out.String()) + } + if strings.Contains(logs.String(), "failed to write DRS map entry") { + t.Fatalf("DRS URI pointer must not be written to the SHA256 map: %s", logs.String()) + } +} diff --git a/internal/gitrepo/gitattributes_routes.go b/internal/gitrepo/gitattributes_routes.go index 9f954138..d7aa2d17 100644 --- a/internal/gitrepo/gitattributes_routes.go +++ b/internal/gitrepo/gitattributes_routes.go @@ -53,7 +53,7 @@ func UpsertDRSRouteLines(gitattributesPath string, mode string, patterns []strin } for _, pat := range order { - newLine := fmt.Sprintf("%s drs.route=%s", pat, mode) + newLine := fmt.Sprintf("%s drs=%s", pat, mode) if idx, ok := seen[pat]; ok { if strings.TrimSpace(lines[idx]) != newLine { lines[idx] = newLine @@ -96,8 +96,11 @@ func parseRouteLine(line string) (pattern string, mode string, ok bool) { pat := parts[0] for _, tok := range parts[1:] { - if strings.HasPrefix(tok, "drs.route=") { - val := strings.TrimPrefix(tok, "drs.route=") + // Accept the former drs.route spelling so existing rules are updated to + // the canonical drs attribute rather than duplicated. + if strings.HasPrefix(tok, "drs=") || strings.HasPrefix(tok, "drs.route=") { + val := strings.TrimPrefix(tok, "drs=") + val = strings.TrimPrefix(val, "drs.route=") val = strings.ToLower(strings.TrimSpace(val)) if val == "ro" || val == "rw" { return pat, val, true diff --git a/internal/gitrepo/gitattributes_routes_test.go b/internal/gitrepo/gitattributes_routes_test.go index c855c03a..d58717ad 100644 --- a/internal/gitrepo/gitattributes_routes_test.go +++ b/internal/gitrepo/gitattributes_routes_test.go @@ -20,7 +20,7 @@ func TestUpsertDRSRouteLinesAddNew(t *testing.T) { b, _ := os.ReadFile(p) got := string(b) - want := "scratch/** drs.route=rw\n" + want := "scratch/** drs=rw\n" if got != want { t.Fatalf("got:\n%q\nwant:\n%q", got, want) } @@ -30,7 +30,7 @@ func TestUpsertDRSRouteLinesUpdateExisting(t *testing.T) { dir := t.TempDir() p := filepath.Join(dir, ".gitattributes") - if err := os.WriteFile(p, []byte("# hi\nscratch/** drs.route=ro\n"), 0o644); err != nil { + if err := os.WriteFile(p, []byte("# hi\nscratch/** drs=ro\n"), 0o644); err != nil { t.Fatal(err) } @@ -44,7 +44,7 @@ func TestUpsertDRSRouteLinesUpdateExisting(t *testing.T) { b, _ := os.ReadFile(p) got := string(b) - want := "# hi\nscratch/** drs.route=rw\n" + want := "# hi\nscratch/** drs=rw\n" if got != want { t.Fatalf("got:\n%q\nwant:\n%q", got, want) } @@ -54,7 +54,7 @@ func TestUpsertDRSRouteLinesIdempotent(t *testing.T) { dir := t.TempDir() p := filepath.Join(dir, ".gitattributes") - if err := os.WriteFile(p, []byte("scratch/** drs.route=rw\n"), 0o644); err != nil { + if err := os.WriteFile(p, []byte("scratch/** drs=rw\n"), 0o644); err != nil { t.Fatal(err) } @@ -68,7 +68,7 @@ func TestUpsertDRSRouteLinesIdempotent(t *testing.T) { } func TestParseRouteLine(t *testing.T) { - p, m, ok := parseRouteLine("scratch/** drs.route=rw") + p, m, ok := parseRouteLine("scratch/** drs=rw") if !ok || p != "scratch/**" || m != "rw" { t.Fatalf("unexpected: ok=%v p=%q m=%q", ok, p, m) } diff --git a/internal/gitrepo/gitattributes_tracking.go b/internal/gitrepo/gitattributes_tracking.go index 9180a42c..f14c7aed 100644 --- a/internal/gitrepo/gitattributes_tracking.go +++ b/internal/gitrepo/gitattributes_tracking.go @@ -12,10 +12,14 @@ import ( ) func TrackPatterns(_ context.Context, patterns []string, verbose bool, dryRun bool) (string, error) { + return trackPatternsAtPath(patterns, verbose, dryRun, ".gitattributes") +} + +func trackPatternsAtPath(patterns []string, verbose bool, dryRun bool, attributesPath string) (string, error) { changedAttribLines := make(map[string]string, len(patterns)) var output strings.Builder - attribContents, err := readLocalGitAttributes() + attribContents, err := readGitAttributes(attributesPath) if err != nil { return "", fmt.Errorf("git drs track failed: %w", err) } @@ -43,7 +47,7 @@ func TrackPatterns(_ context.Context, patterns []string, verbose bool, dryRun bo } if !dryRun { - if err := writeMergedGitAttributes(attribContents, changedAttribLines, false); err != nil { + if err := writeMergedGitAttributes(attributesPath, attribContents, changedAttribLines, false); err != nil { return "", fmt.Errorf("git drs track failed: %w", err) } } @@ -143,16 +147,16 @@ func UntrackPatterns(_ context.Context, patterns []string, _ bool, dryRun bool) } func TrackReadOnly(ctx context.Context, path string) (bool, error) { - if _, err := TrackPatterns(ctx, []string{path}, false, false); err != nil { - return false, fmt.Errorf("git lfs track failed: %w", err) - } - repoRoot, err := GitTopLevel() if err != nil { return false, err } attrPath := filepath.Join(repoRoot, ".gitattributes") + if _, err := trackPatternsAtPath([]string{path}, false, false, attrPath); err != nil { + return false, fmt.Errorf("git lfs track failed: %w", err) + } + changed, err := UpsertDRSRouteLines(attrPath, "ro", []string{path}) if err != nil { return false, err @@ -162,7 +166,11 @@ func TrackReadOnly(ctx context.Context, path string) (bool, error) { } func readLocalGitAttributes() ([]byte, error) { - data, err := os.ReadFile(".gitattributes") + return readGitAttributes(".gitattributes") +} + +func readGitAttributes(attributesPath string) ([]byte, error) { + data, err := os.ReadFile(attributesPath) if err != nil { if os.IsNotExist(err) { return nil, nil @@ -189,7 +197,7 @@ func parseKnownLFSPatterns(content []byte) map[string]string { return known } -func writeMergedGitAttributes(existing []byte, changed map[string]string, dryRun bool) error { +func writeMergedGitAttributes(attributesPath string, existing []byte, changed map[string]string, dryRun bool) error { if dryRun { return nil } @@ -223,7 +231,7 @@ func writeMergedGitAttributes(existing []byte, changed map[string]string, dryRun if content != "" { content += "\n" } - if err := os.WriteFile(".gitattributes", []byte(content), 0o644); err != nil { + if err := os.WriteFile(attributesPath, []byte(content), 0o644); err != nil { return fmt.Errorf("write .gitattributes: %w", err) } return nil diff --git a/internal/gitrepo/gitattributes_tracking_test.go b/internal/gitrepo/gitattributes_tracking_test.go index cc9ca3df..82d237eb 100644 --- a/internal/gitrepo/gitattributes_tracking_test.go +++ b/internal/gitrepo/gitattributes_tracking_test.go @@ -3,6 +3,7 @@ package gitrepo import ( "context" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -51,6 +52,40 @@ func TestTrackPatternsDryRunDoesNotWrite(t *testing.T) { } } +func TestTrackReadOnlyWritesRootAttributesFromSubdirectory(t *testing.T) { + repo := t.TempDir() + cmd := exec.Command("git", "init") + cmd.Dir = repo + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git init: %v: %s", err, output) + } + subdir := filepath.Join(repo, "nested") + if err := os.Mkdir(subdir, 0o755); err != nil { + t.Fatal(err) + } + oldwd := mustChdirTrackTest(t, subdir) + t.Cleanup(func() { _ = os.Chdir(oldwd) }) + + const path = "data/pointer" + if _, err := TrackReadOnly(context.Background(), path); err != nil { + t.Fatalf("TrackReadOnly: %v", err) + } + if _, err := os.Stat(filepath.Join(subdir, ".gitattributes")); !os.IsNotExist(err) { + t.Fatalf("expected no nested .gitattributes, stat err=%v", err) + } + contents, err := os.ReadFile(filepath.Join(repo, ".gitattributes")) + if err != nil { + t.Fatalf("read root .gitattributes: %v", err) + } + got := string(contents) + if !strings.Contains(got, "data/pointer filter=drs diff=drs merge=drs -text") { + t.Fatalf("root .gitattributes does not track pointer: %q", got) + } + if !strings.Contains(got, "data/pointer drs=ro") { + t.Fatalf("root .gitattributes does not mark pointer read-only: %q", got) + } +} + func TestListTrackedPatternsReadsGitattributes(t *testing.T) { repo := t.TempDir() oldwd := mustChdirTrackTest(t, repo) diff --git a/internal/gitrepo/paths.go b/internal/gitrepo/paths.go index ab2669e2..0eeac868 100644 --- a/internal/gitrepo/paths.go +++ b/internal/gitrepo/paths.go @@ -4,6 +4,7 @@ const ( LFSObjectsPath = ".git/lfs/objects" DRSObjectsPath = ".git/drs/lfs/objects" ConfigYAML = "config.yaml" + RepoDRSDir = ".git-drs" DRSLogFile = ".git/drs/git-drs.log" DRSDir = ".git/drs" ) diff --git a/internal/lfs/inventory.go b/internal/lfs/inventory.go index 259b0f29..fcf89ca5 100644 --- a/internal/lfs/inventory.go +++ b/internal/lfs/inventory.go @@ -10,10 +10,10 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "strconv" "strings" + "github.com/calypr/git-drs/internal/drsobject" drsapi "github.com/calypr/syfon/apigen/client/drs" "github.com/calypr/syfon/client/hash" ) @@ -28,6 +28,7 @@ type LfsFileInfo struct { OidType string `json:"oid_type"` Oid string `json:"oid"` Version string `json:"version"` + SHA256 string `json:"sha256,omitempty"` } func IsLFSTracked(path string) (bool, error) { @@ -145,6 +146,7 @@ func GetWorktreeLfsFiles(logger *slog.Logger) (map[string]LfsFileInfo, error) { OidType: pointer.OidType, Oid: pointer.Oid, Version: pointer.Version, + SHA256: pointer.SHA256, } } return files, nil @@ -214,6 +216,7 @@ func addFilesFromPaths(ctx context.Context, repoDir, ref string, paths []string, OidType: pointer.OidType, Oid: pointer.Oid, Version: pointer.Version, + SHA256: pointer.SHA256, } } @@ -410,6 +413,7 @@ func readWorktreePointerInfo(repoDir, path string) (LfsFileInfo, bool) { OidType: pointer.OidType, Oid: pointer.Oid, Version: pointer.Version, + SHA256: pointer.SHA256, }, true } @@ -429,11 +433,12 @@ func readIndexPointerInfo(ctx context.Context, repoDir, path string) (LfsFileInf OidType: pointer.OidType, Oid: pointer.Oid, Version: pointer.Version, + SHA256: pointer.SHA256, }, true } func grepPointerPaths(ctx context.Context, repoDir, ref string) ([]string, error) { - cmd := exec.CommandContext(ctx, "git", "grep", "-z", "-l", "https://git-lfs.github.com/spec/v1", ref, "--") + cmd := exec.CommandContext(ctx, "git", "grep", "-z", "-l", "-e", "https://git-lfs.github.com/spec/v1", "-e", "https://calypr.github.io/spec/v1", ref, "--") cmd.Dir = repoDir var stdout, stderr bytes.Buffer cmd.Stdout = &stdout @@ -489,12 +494,11 @@ type lfsPointer struct { OidType string Oid string Size int64 + SHA256 string } func parseLFSPointer(content string) (lfsPointer, bool) { var p lfsPointer - sha256Re := regexp.MustCompile(`(?i)^[a-f0-9]{64}$`) - for _, line := range strings.Split(strings.ReplaceAll(content, "\r\n", "\n"), "\n") { line = strings.TrimSpace(line) if line == "" { @@ -521,13 +525,32 @@ func parseLFSPointer(content string) (lfsPointer, bool) { return lfsPointer{}, false } p.Size = sz + continue + } + if strings.HasPrefix(line, "sha256 ") { + p.SHA256 = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(line, "sha256 "))) + if !sha256OIDRe.MatchString(p.SHA256) { + return lfsPointer{}, false + } } } if p.Version == "" || p.OidType == "" || p.Oid == "" { return lfsPointer{}, false } - if p.OidType != "sha256" || !sha256Re.MatchString(p.Oid) { + switch strings.ToLower(p.OidType) { + case "sha256": + p.OidType = "sha256" + if !sha256OIDRe.MatchString(p.Oid) { + return lfsPointer{}, false + } + p.Oid = strings.ToLower(p.Oid) + case "drs": + p.OidType = "drs" + if p.Version != "https://calypr.github.io/spec/v1" || !strings.HasPrefix(p.Oid, "//") { + return lfsPointer{}, false + } + default: return lfsPointer{}, false } @@ -551,17 +574,40 @@ func CreateLfsPointer(drsObj *drsapi.DrsObject, dst string) error { if shaSum == "" { return fmt.Errorf("no sha256 checksum found for DRS object") } + return CreateLfsPointerWithOID(drsObj, dst, shaSum) +} - // create pointer file content +// CreateLfsPointerWithOID writes a Git LFS pointer using an explicit local/cache +// oid. The oid may be a real content SHA256 or a derived source-identity key. +func CreateLfsPointerWithOID(drsObj *drsapi.DrsObject, dst string, oid string) error { + oid = strings.TrimPrefix(strings.TrimSpace(oid), "sha256:") + if !sha256OIDRe.MatchString(oid) { + return fmt.Errorf("oid %q is not a valid sha256-shaped value", oid) + } pointerContent := "version https://git-lfs.github.com/spec/v1\n" - pointerContent += fmt.Sprintf("oid sha256:%s\n", shaSum) + pointerContent += fmt.Sprintf("oid sha256:%s\n", strings.ToLower(oid)) pointerContent += fmt.Sprintf("size %d\n", drsObj.Size) - - // write to file - err := os.WriteFile(dst, []byte(pointerContent), 0644) - if err != nil { + if err := os.WriteFile(dst, []byte(pointerContent), 0644); err != nil { return fmt.Errorf("failed to write LFS pointer file: %w", err) } + return nil +} +// CreateDRSPointer writes a git-drs pointer that preserves the retrievable DRS URI. +func CreateDRSPointer(drsObj *drsapi.DrsObject, dst string, drsURI string) error { + drsURI = strings.TrimSpace(drsURI) + if !strings.HasPrefix(strings.ToLower(drsURI), "drs://") { + return fmt.Errorf("DRS URI %q must start with drs://", drsURI) + } + pointerOID := "//" + drsURI[len("drs://"):] + pointerContent := "version https://calypr.github.io/spec/v1\n" + pointerContent += fmt.Sprintf("oid drs:%s\n", pointerOID) + pointerContent += fmt.Sprintf("size %d\n", drsObj.Size) + if checksum := drsobject.NormalizeChecksum(hash.ConvertDrsChecksumsToHashInfo(drsObj.Checksums).SHA256); checksum != "" { + pointerContent += fmt.Sprintf("sha256 %s\n", strings.ToLower(checksum)) + } + if err := os.WriteFile(dst, []byte(pointerContent), 0644); err != nil { + return fmt.Errorf("failed to write DRS pointer file: %w", err) + } return nil } diff --git a/internal/lfs/object_path.go b/internal/lfs/object_path.go index 7085cbef..c2031959 100644 --- a/internal/lfs/object_path.go +++ b/internal/lfs/object_path.go @@ -1,17 +1,60 @@ package lfs import ( + "crypto/sha256" + "encoding/hex" "fmt" "path/filepath" + "regexp" "strings" ) -// ObjectPath returns the Git LFS fanout path for a sha256 object ID. +var sha256OIDRe = regexp.MustCompile(`(?i)^[a-f0-9]{64}$`) + +// ObjectPath returns the Git LFS fanout path for an object ID. +// +// The cache layout remains SHA256-shaped for compatibility with existing LFS +// object storage. Git LFS pointers pass their sha256 object IDs through +// unchanged, while git-drs DRS URI pointers are assigned a deterministic local +// cache key derived from the normalized DRS URI. func ObjectPath(basePath string, oid string) (string, error) { + cacheKey, err := cacheKeyForOID(oid) + if err != nil { + return "", err + } + + return filepath.Join(basePath, cacheKey[:2], cacheKey[2:4], cacheKey), nil +} + +func cacheKeyForOID(oid string) (string, error) { + oid = strings.TrimSpace(oid) oid = strings.TrimPrefix(oid, "sha256:") - if len(oid) != 64 { - return "", fmt.Errorf("error: %s is not a valid sha256 hash", oid) + if sha256OIDRe.MatchString(oid) { + return strings.ToLower(oid), nil + } + + if IsDRSURI(oid) { + sum := sha256.Sum256([]byte("git-drs-anvil-ref:v1\n" + normalizeDRSURI(oid))) + return hex.EncodeToString(sum[:]), nil } - return filepath.Join(basePath, oid[:2], oid[2:4], oid), nil + return "", fmt.Errorf("error: %s is not a valid sha256 hash or DRS URI", oid) +} + +// IsDRSURI reports whether uri is a canonical DRS URI or the //authority/path +// form stored internally after parsing a git-drs pointer's "oid drs:" line. +func IsDRSURI(uri string) bool { + uri = strings.TrimSpace(uri) + return strings.HasPrefix(strings.ToLower(uri), "drs://") || strings.HasPrefix(uri, "//") +} + +func normalizeDRSURI(uri string) string { + uri = strings.TrimSpace(uri) + if strings.HasPrefix(uri, "//") { + return "drs:" + uri + } + if len(uri) >= len("drs://") && strings.EqualFold(uri[:len("drs://")], "drs://") { + return "drs://" + uri[len("drs://"):] + } + return uri } diff --git a/internal/lfs/object_path_test.go b/internal/lfs/object_path_test.go index 4c918172..e0ab6473 100644 --- a/internal/lfs/object_path_test.go +++ b/internal/lfs/object_path_test.go @@ -8,3 +8,17 @@ func TestObjectPathRejectsInvalidOID(t *testing.T) { t.Fatalf("expected validation error, got %s", path) } } + +func TestObjectPathAcceptsDRSPointerOIDValue(t *testing.T) { + fullURIPath, err := ObjectPath(".git/drs/objects", "drs://cgc-ga4gh-api.sbgenomics.com/4c33ae65e4b08832ce3d94e9c") + if err != nil { + t.Fatalf("expected DRS URI object path: %v", err) + } + pointerOIDPath, err := ObjectPath(".git/drs/objects", "//cgc-ga4gh-api.sbgenomics.com/4c33ae65e4b08832ce3d94e9c") + if err != nil { + t.Fatalf("expected DRS pointer oid object path: %v", err) + } + if pointerOIDPath != fullURIPath { + t.Fatalf("expected DRS pointer oid and full URI to produce same path, got %q and %q", pointerOIDPath, fullURIPath) + } +} diff --git a/internal/lfs/terra_pointer_acceptance_test.go b/internal/lfs/terra_pointer_acceptance_test.go new file mode 100644 index 00000000..b9b72473 --- /dev/null +++ b/internal/lfs/terra_pointer_acceptance_test.go @@ -0,0 +1,47 @@ +package lfs + +import "testing" + +func TestAcceptanceTerraDRSURIoidPointerIsRecognized(t *testing.T) { + const pointer = `version https://calypr.github.io/spec/v1 +oid drs://cgc-ga4gh-api.sbgenomics.com/4c33ae65e4b08832ce3d94e9c +size 11305017366 +` + + info, ok := parseLFSPointer(pointer) + if !ok { + t.Fatalf("expected git-drs DRS URI pointer to be recognized") + } + if info.Version != "https://calypr.github.io/spec/v1" { + t.Fatalf("unexpected pointer version: %q", info.Version) + } + if info.OidType != "drs" || info.Oid != "//cgc-ga4gh-api.sbgenomics.com/4c33ae65e4b08832ce3d94e9c" { + t.Fatalf("expected DRS URI oid to be preserved, got type=%q oid=%q", info.OidType, info.Oid) + } + if info.Size != 11305017366 { + t.Fatalf("unexpected pointer size: %d", info.Size) + } +} + +func TestAnVILCacheIdentityGolden(t *testing.T) { + got, err := cacheKeyForOID("DRS://example.org/object-1") + if err != nil { + t.Fatal(err) + } + const want = "d644000bac79ca85d486629f9271ebf2da9884b7bca0d8ad1ee514f9241cbc74" + if got != want { + t.Fatalf("cache identity changed: got %s, want %s", got, want) + } +} + +func TestAcceptanceTerraDRSURIoidPointerHasDeterministicSHA256CacheKey(t *testing.T) { + const drsURI = "drs://cgc-ga4gh-api.sbgenomics.com/4c33ae65e4b08832ce3d94e9c" + + cacheKeyPath, err := ObjectPath(".git/drs/objects", drsURI) + if err != nil { + t.Fatalf("expected DRS URI to be normalized to a sha256-shaped cache key path: %v", err) + } + if cacheKeyPath == "" { + t.Fatalf("expected non-empty cache path for DRS URI") + } +} diff --git a/internal/presets/presets.go b/internal/presets/presets.go new file mode 100644 index 00000000..891ddb44 --- /dev/null +++ b/internal/presets/presets.go @@ -0,0 +1,67 @@ +// Package presets exposes the non-secret remote defaults shipped with git-drs. +package presets + +import ( + _ "embed" + "fmt" + "strings" + + "gopkg.in/yaml.v3" +) + +// CatalogVersion is persisted when a preset is expanded. +const CatalogVersion = 1 + +type Preset struct { + Alias string `yaml:"alias"` + Endpoint string `yaml:"endpoint"` + Provider string `yaml:"provider"` + Auth string `yaml:"auth"` + RegistryServiceID string `yaml:"registry_service_id,omitempty"` +} + +type catalog struct { + Version int `yaml:"version"` + Presets []Preset `yaml:"presets"` +} + +//go:embed presets.yaml +var builtIn []byte + +func List() ([]Preset, error) { + return loadCatalog(builtIn) +} + +func loadCatalog(data []byte) ([]Preset, error) { + var c catalog + if err := yaml.Unmarshal(data, &c); err != nil { + return nil, fmt.Errorf("decode built-in preset catalog: %w", err) + } + if c.Version != CatalogVersion { + return nil, fmt.Errorf("unsupported built-in preset catalog version %d", c.Version) + } + for i, preset := range c.Presets { + if strings.TrimSpace(preset.Auth) == "" { + name := strings.TrimSpace(preset.Alias) + if name == "" { + name = fmt.Sprintf("at index %d", i) + } + return nil, fmt.Errorf("preset %q does not specify an authentication type", name) + } + } + return append([]Preset(nil), c.Presets...), nil +} + +func Lookup(alias string) (Preset, bool, error) { + alias = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(alias)), "built-in:") + items, err := List() + if err != nil { + return Preset{}, false, err + } + for _, item := range items { + if item.Alias == alias { + return item, true, nil + } + } + return Preset{}, false, nil +} diff --git a/internal/presets/presets.yaml b/internal/presets/presets.yaml new file mode 100644 index 00000000..4d3a6bfe --- /dev/null +++ b/internal/presets/presets.yaml @@ -0,0 +1,20 @@ +version: 1 +presets: + - alias: calypr + endpoint: https://calypr.org + provider: gen3 + auth: provider-helper:gen3-profile + - alias: terra + endpoint: https://data.terra.bio + provider: terra + auth: google-adc + - alias: synapse + endpoint: https://repo-prod.prod.sagebase.org/ga4gh/drs/v1 + provider: synapse + auth: bearer + registry_service_id: org.sagebase.prod.repo-prod + - alias: cgc + endpoint: https://cgc-ga4gh-api.sbgenomics.com/ga4gh/drs/v1 + provider: cgc + auth: bearer + registry_service_id: com.sb.cgc.drs diff --git a/internal/presets/presets_test.go b/internal/presets/presets_test.go new file mode 100644 index 00000000..a58c118f --- /dev/null +++ b/internal/presets/presets_test.go @@ -0,0 +1,38 @@ +package presets + +import ( + "strings" + "testing" +) + +func TestBuiltInCatalog(t *testing.T) { + items, err := List() + if err != nil { + t.Fatal(err) + } + if len(items) != 4 { + t.Fatalf("got %d presets, want 4", len(items)) + } + for _, alias := range []string{"calypr", "terra", "synapse", "cgc"} { + p, ok, err := Lookup(alias) + if err != nil || !ok || p.Endpoint == "" || p.Provider == "" || p.Auth == "" { + t.Fatalf("invalid %s preset: %+v, %v", alias, p, err) + } + } +} + +func TestCatalogRequiresPresetAuthenticationType(t *testing.T) { + _, err := loadCatalog([]byte(` +version: 1 +presets: + - alias: unauthenticated + endpoint: https://drs.example.org + provider: ga4gh +`)) + if err == nil { + t.Fatal("expected a preset without an authentication type to be rejected") + } + if !strings.Contains(err.Error(), `preset "unauthenticated" does not specify an authentication type`) { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/internal/remoteruntime/bucket_lookup.go b/internal/remoteruntime/bucket_lookup.go new file mode 100644 index 00000000..2994519c --- /dev/null +++ b/internal/remoteruntime/bucket_lookup.go @@ -0,0 +1,90 @@ +package remoteruntime + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "slices" + "strings" + + "github.com/calypr/git-drs/internal/gitrepo" + bucketapi "github.com/calypr/syfon/apigen/client/bucketapi" + syfoncommon "github.com/calypr/syfon/common" +) + +// resolveBucketScopeFromServer discovers the bucket exposed for a Gen3 scope. +// This is the runtime fallback for remotes configured with a scope but without +// an explicit storage location or repository-local bucket mapping. +func resolveBucketScopeFromServer(ctx context.Context, endpoint, token, organization, project, preferredBucket string) (gitrepo.ResolvedBucketScope, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(endpoint, "/")+"/data/buckets", nil) + if err != nil { + return gitrepo.ResolvedBucketScope{}, fmt.Errorf("build bucket list request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(token)) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return gitrepo.ResolvedBucketScope{}, fmt.Errorf("request bucket list: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return gitrepo.ResolvedBucketScope{}, fmt.Errorf("bucket list failed with status %d", resp.StatusCode) + } + + var payload bucketapi.BucketsResponse + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + return gitrepo.ResolvedBucketScope{}, fmt.Errorf("decode bucket list response: %w", err) + } + + bucket, err := resolveBucketFromPayload(payload, organization, project, preferredBucket) + if err != nil { + return gitrepo.ResolvedBucketScope{}, err + } + return gitrepo.ResolvedBucketScope{Bucket: bucket}, nil +} + +func resolveBucketFromPayload(payload bucketapi.BucketsResponse, organization, project, preferredBucket string) (string, error) { + projectResource, err := syfoncommon.ResourcePath(organization, project) + if err != nil { + return "", err + } + orgResource, err := syfoncommon.ResourcePath(organization, "") + if err != nil { + return "", err + } + + for _, resource := range []string{projectResource, orgResource} { + matches := make([]string, 0) + for bucket, meta := range payload.S3BUCKETS { + if meta.Programs == nil { + continue + } + for _, candidate := range *meta.Programs { + if syfoncommon.NormalizeAccessResource(candidate) == syfoncommon.NormalizeAccessResource(resource) { + matches = append(matches, bucket) + break + } + } + } + slices.Sort(matches) + matches = slices.Compact(matches) + if len(matches) == 0 { + continue + } + if preferredBucket != "" { + for _, bucket := range matches { + if strings.EqualFold(bucket, strings.TrimSpace(preferredBucket)) { + return bucket, nil + } + } + return "", fmt.Errorf("selected bucket %q does not match resource %q; choose one of: %s", preferredBucket, resource, strings.Join(matches, ", ")) + } + if len(matches) == 1 { + return matches[0], nil + } + return "", fmt.Errorf("multiple visible server buckets matched resource %q: %s", resource, strings.Join(matches, ", ")) + } + + return "", fmt.Errorf("no visible server bucket matched organization=%q project=%q", organization, project) +} diff --git a/internal/remoteruntime/runtime.go b/internal/remoteruntime/runtime.go index 39410cdc..b0832fc1 100644 --- a/internal/remoteruntime/runtime.go +++ b/internal/remoteruntime/runtime.go @@ -5,6 +5,9 @@ import ( "fmt" "log/slog" "net/url" + "os" + "os/exec" + "path/filepath" "strings" "time" @@ -20,6 +23,8 @@ const credentialHelpSuffix = "Refresh credentials with `git drs remote add gen3 type GitContext struct { Client *syclient.Client + RemoteType config.RemoteType + Endpoint string Organization string ProjectId string BucketName string @@ -30,8 +35,21 @@ type GitContext struct { UploadConcurrency int Logger *slog.Logger Credential *syconf.Credential + Capabilities Capabilities } +// Capabilities is the command-facing contract for a resolved remote. Commands +// must use this contract rather than infer behaviour from configuration types. +type Capabilities struct { + Resolve, Download, Upload, Register, ReadOnly bool +} + +func (g *GitContext) CanResolve() bool { return g != nil && g.Capabilities.Resolve } +func (g *GitContext) CanDownload() bool { return g != nil && g.Capabilities.Download } +func (g *GitContext) CanUpload() bool { return g != nil && g.Capabilities.Upload } +func (g *GitContext) CanRegister() bool { return g != nil && g.Capabilities.Register } +func (g *GitContext) IsReadOnly() bool { return g != nil && g.Capabilities.ReadOnly } + func New(cfg *config.Config, remote config.Remote, logger *slog.Logger) (*GitContext, error) { x, ok := cfg.Remotes[remote] if !ok { @@ -47,12 +65,56 @@ func New(cfg *config.Config, remote config.Remote, logger *slog.Logger) (*GitCon } return gen3Client(string(remote), *x.Gen3, logger) } + if x.Terra != nil { + return terraClient(*x.Terra, logger) + } + if x.Generic != nil { + switch x.Generic.Provider { + case "terra": + return terraClient(config.TerraRemote{Endpoint: x.Generic.Endpoint, Auth: x.Generic.Auth, Mode: "read-only"}, logger) + case "gen3": + switch x.Generic.Auth { + case "bearer", "provider-helper", "provider-helper:gen3-profile": + return gen3ClientWithCredential(string(remote), x.Generic.Credential, config.Gen3Remote{ + Endpoint: x.Generic.Endpoint, Organization: x.Generic.GetOrganization(), + ProjectID: x.Generic.GetProjectId(), Bucket: x.Generic.GetBucketName(), + StoragePrefix: x.Generic.GetStoragePrefix(), + }, logger) + default: + return nil, fmt.Errorf("authentication method %q is not supported by the Gen3 remote adapter", x.Generic.Auth) + } + case "ga4gh", "auto", "cgc", "synapse": + return nil, fmt.Errorf("provider %q does not yet have an operational remote adapter", x.Generic.Provider) + default: + return nil, fmt.Errorf("unsupported remote provider %q", x.Generic.Provider) + } + } return nil, fmt.Errorf("no valid remote configuration found for current remote: %s", remote) } +func terraClient(remote config.TerraRemote, logger *slog.Logger) (*GitContext, error) { + if strings.TrimSpace(remote.Endpoint) == "" { + return nil, fmt.Errorf("no terra endpoint specified") + } + if _, err := url.Parse(remote.Endpoint); err != nil { + return nil, err + } + return &GitContext{ + RemoteType: config.TerraServerType, + Endpoint: remote.Endpoint, + Logger: logger, + Credential: &syconf.Credential{APIEndpoint: remote.Endpoint}, + Capabilities: Capabilities{Resolve: true, Download: true, ReadOnly: true}, + }, nil +} + func gen3Client(remoteName string, remote config.Gen3Remote, logger *slog.Logger) (*GitContext, error) { + return gen3ClientWithCredential(remoteName, "", remote, logger) +} + +func gen3ClientWithCredential(remoteName, source string, remote config.Gen3Remote, logger *slog.Logger) (*GitContext, error) { manager := calyprconf.NewConfigure(logger) - cred, err := manager.Load(remoteName) + cred, saveRefreshed, err := resolveGen3Credential(manager, source, remoteName, remote.Endpoint) if err != nil { return nil, err } @@ -61,12 +123,81 @@ func gen3Client(remoteName string, remote config.Gen3Remote, logger *slog.Logger if err := credentials.EnsureValidCredential(ctx, cred, logger); err != nil { return nil, WrapCredentialValidationError(remoteName, err) } - if err := manager.Save(cred); err != nil { - return nil, fmt.Errorf("save refreshed credential for remote %q: %w", remoteName, err) + if saveRefreshed { + if err := manager.Save(cred); err != nil { + return nil, fmt.Errorf("save refreshed credential for remote %q: %w", remoteName, err) + } } return newGitContext(*cred, remote, logger) } +type gen3CredentialManager interface { + Import(filePath, fenceToken string) (*calyprconf.Credential, error) + Load(profile string) (*calyprconf.Credential, error) +} + +// resolveGen3Credential turns the clone-local source selector into credential +// material. The configured remote endpoint remains authoritative: a credential +// source must not be able to redirect requests to another host. +func resolveGen3Credential(manager gen3CredentialManager, source, remoteName, endpoint string) (*calyprconf.Credential, bool, error) { + source = strings.TrimSpace(source) + if source == "" { + cred, err := manager.Load(remoteName) + return cred, true, err + } + + var cred *calyprconf.Credential + var err error + switch { + case strings.HasPrefix(source, "profile:"): + profile := strings.TrimSpace(strings.TrimPrefix(source, "profile:")) + cred, err = manager.Load(profile) + if err == nil { + cred.Profile = profile + } + if err != nil { + return nil, false, fmt.Errorf("load Gen3 credential profile %q: %w", profile, err) + } + cred.APIEndpoint = endpoint + return cred, true, nil + case strings.HasPrefix(source, "file:"): + fileName := strings.TrimSpace(strings.TrimPrefix(source, "file:")) + if strings.HasPrefix(fileName, "~/") { + if home, homeErr := os.UserHomeDir(); homeErr == nil { + fileName = filepath.Join(home, strings.TrimPrefix(fileName, "~/")) + } + } + cred, err = manager.Import(fileName, "") + if err != nil { + return nil, false, fmt.Errorf("import Gen3 credential file %q: %w", fileName, err) + } + case strings.HasPrefix(source, "env:"): + name := strings.TrimSpace(strings.TrimPrefix(source, "env:")) + token, ok := os.LookupEnv(name) + if !ok || strings.TrimSpace(token) == "" { + return nil, false, fmt.Errorf("Gen3 credential environment variable %q is empty or unset", name) + } + cred = &calyprconf.Credential{AccessToken: strings.TrimSpace(token)} + case strings.HasPrefix(source, "helper:"): + name := strings.TrimSpace(strings.TrimPrefix(source, "helper:")) + output, helperErr := exec.Command(name).Output() + if helperErr != nil { + return nil, false, fmt.Errorf("run Gen3 credential helper %q: %w", name, helperErr) + } + token := strings.TrimSpace(string(output)) + if token == "" { + return nil, false, fmt.Errorf("Gen3 credential helper %q returned an empty token", name) + } + cred = &calyprconf.Credential{AccessToken: token} + default: + return nil, false, fmt.Errorf("unsupported Gen3 credential source %q", source) + } + + cred.Profile = remoteName + cred.APIEndpoint = endpoint + return cred, false, nil +} + func localClient(remoteName string, remote config.LocalRemote, logger *slog.Logger) (*GitContext, error) { if username, password, err := gitrepo.GetRemoteBasicAuth(remoteName); err == nil && username != "" && password != "" { remote.BasicUsername = username @@ -106,12 +237,15 @@ func localClient(remoteName string, remote config.LocalRemote, logger *slog.Logg return &GitContext{ Client: client, + RemoteType: config.LocalServerType, + Endpoint: remote.BaseURL, Organization: remote.GetOrganization(), ProjectId: projectID, BucketName: bucketName, StoragePrefix: storagePrefix, Logger: logger, Credential: cred, + Capabilities: Capabilities{Resolve: true, Download: true, Upload: true, Register: true}, }, nil } @@ -131,7 +265,19 @@ func newGitContext(profileConfig syconf.Credential, remote config.Gen3Remote, lo remote.GetStoragePrefix(), ) if err != nil { - return nil, err + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + scope, err = resolveBucketScopeFromServer( + ctx, + profileConfig.APIEndpoint, + profileConfig.AccessToken, + remote.GetOrganization(), + projectID, + remote.GetBucketName(), + ) + if err != nil { + return nil, fmt.Errorf("failed resolving bucket mapping for organization=%q project=%q: %w", remote.GetOrganization(), projectID, err) + } } raw, err := syclient.New(profileConfig.APIEndpoint, syclient.WithBearerToken(profileConfig.AccessToken)) @@ -150,6 +296,8 @@ func newGitContext(profileConfig syconf.Credential, remote config.Gen3Remote, lo return &GitContext{ Client: client, + RemoteType: config.Gen3ServerType, + Endpoint: profileConfig.APIEndpoint, ProjectId: projectID, BucketName: scope.Bucket, Organization: remote.GetOrganization(), @@ -159,6 +307,7 @@ func newGitContext(profileConfig syconf.Credential, remote config.Gen3Remote, lo UploadConcurrency: uploadConcurrency, Logger: logger, Credential: &profileConfig, + Capabilities: Capabilities{Resolve: true, Download: true, Upload: true, Register: true}, }, nil } diff --git a/internal/remoteruntime/runtime_test.go b/internal/remoteruntime/runtime_test.go index 2b4d6720..ee19fa25 100644 --- a/internal/remoteruntime/runtime_test.go +++ b/internal/remoteruntime/runtime_test.go @@ -1,8 +1,12 @@ package remoteruntime import ( + "errors" + "net/http" + "net/http/httptest" "os" "os/exec" + "path/filepath" "testing" "github.com/calypr/git-drs/internal/config" @@ -11,6 +15,31 @@ import ( syconf "github.com/calypr/syfon/client/config" ) +type fakeGen3CredentialManager struct { + importPath string + loadName string + imported *syconf.Credential + loaded *syconf.Credential +} + +func (m *fakeGen3CredentialManager) Import(filePath, _ string) (*syconf.Credential, error) { + m.importPath = filePath + if m.imported == nil { + return nil, errors.New("unexpected import") + } + copy := *m.imported + return ©, nil +} + +func (m *fakeGen3CredentialManager) Load(profile string) (*syconf.Credential, error) { + m.loadName = profile + if m.loaded == nil { + return nil, errors.New("unexpected load") + } + copy := *m.loaded + return ©, nil +} + func setupTestRepo(t *testing.T) string { t.Helper() @@ -59,6 +88,138 @@ func TestNewLocalIncludesRepoBasicAuth(t *testing.T) { } } +func TestNewTerraRemoteContext(t *testing.T) { + setupTestRepo(t) + + cfg, err := config.UpdateRemote(config.Remote("anvil"), config.RemoteSelect{ + Terra: &config.TerraRemote{ + Endpoint: "https://data.terra.bio", + Auth: "google-adc", + Mode: "read-only", + }, + }) + if err != nil { + t.Fatalf("UpdateRemote failed: %v", err) + } + + gitCtx, err := New(cfg, config.Remote("anvil"), drslog.GetLogger()) + if err != nil { + t.Fatalf("New failed: %v", err) + } + if gitCtx.RemoteType != config.TerraServerType { + t.Fatalf("RemoteType = %q, want %q", gitCtx.RemoteType, config.TerraServerType) + } + if gitCtx.Endpoint != "https://data.terra.bio" { + t.Fatalf("Endpoint = %q, want Terra endpoint", gitCtx.Endpoint) + } + if gitCtx.Client != nil { + t.Fatalf("Terra runtime should not create a Syfon client, got %+v", gitCtx.Client) + } + if !gitCtx.CanResolve() || !gitCtx.CanDownload() || !gitCtx.IsReadOnly() || gitCtx.CanUpload() || gitCtx.CanRegister() { + t.Fatalf("unexpected Terra capabilities: %+v", gitCtx.Capabilities) + } +} + +func TestNewGenericTerraUsesProviderAdapter(t *testing.T) { + setupTestRepo(t) + cfg := &config.Config{Remotes: map[config.Remote]config.RemoteSelect{ + "anvil": {Generic: &config.GenericRemote{Endpoint: "https://data.terra.bio", Provider: "terra", Auth: "google-adc"}}, + }} + gitCtx, err := New(cfg, "anvil", drslog.GetLogger()) + if err != nil { + t.Fatal(err) + } + if gitCtx.RemoteType != config.TerraServerType || !gitCtx.IsReadOnly() || !gitCtx.CanDownload() { + t.Fatalf("generic Terra did not resolve through Terra adapter: %+v", gitCtx) + } +} + +func TestResolveGen3CredentialUsesSelectedProfile(t *testing.T) { + manager := &fakeGen3CredentialManager{loaded: &syconf.Credential{ + Profile: "research", APIEndpoint: "https://old.example", APIKey: "key", + }} + cred, save, err := resolveGen3Credential(manager, "profile:research", "production", "https://gen3.example") + if err != nil { + t.Fatal(err) + } + if manager.loadName != "research" || cred.Profile != "research" { + t.Fatalf("selected profile was not loaded: name=%q credential=%+v", manager.loadName, cred) + } + if cred.APIEndpoint != "https://gen3.example" { + t.Fatalf("APIEndpoint = %q, want configured remote endpoint", cred.APIEndpoint) + } + if !save { + t.Fatal("a refreshed profile credential should be saved to its source profile") + } +} + +func TestResolveGen3CredentialImportsFile(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + manager := &fakeGen3CredentialManager{imported: &syconf.Credential{APIKey: "key"}} + cred, save, err := resolveGen3Credential(manager, "file:~/.gen3/credentials.json", "production", "https://gen3.example") + if err != nil { + t.Fatal(err) + } + wantPath := filepath.Join(os.Getenv("HOME"), ".gen3", "credentials.json") + if manager.importPath != wantPath { + t.Fatalf("import path = %q, want %q", manager.importPath, wantPath) + } + if cred.Profile != "production" || cred.APIEndpoint != "https://gen3.example" || save { + t.Fatalf("unexpected imported credential: credential=%+v save=%v", cred, save) + } +} + +func TestResolveGen3CredentialReadsEnvironment(t *testing.T) { + t.Setenv("GEN3_TOKEN", " token-from-env \n") + cred, save, err := resolveGen3Credential(&fakeGen3CredentialManager{}, "env:GEN3_TOKEN", "production", "https://gen3.example") + if err != nil { + t.Fatal(err) + } + if cred.AccessToken != "token-from-env" || cred.APIEndpoint != "https://gen3.example" || save { + t.Fatalf("unexpected environment credential: credential=%+v save=%v", cred, save) + } +} + +func TestResolveGen3CredentialRunsHelper(t *testing.T) { + dir := t.TempDir() + helper := filepath.Join(dir, "gen3-token-helper") + if err := os.WriteFile(helper, []byte("#!/bin/sh\nprintf 'token-from-helper\\n'\n"), 0o700); err != nil { + t.Fatal(err) + } + cred, save, err := resolveGen3Credential(&fakeGen3CredentialManager{}, "helper:"+helper, "production", "https://gen3.example") + if err != nil { + t.Fatal(err) + } + if cred.AccessToken != "token-from-helper" || cred.APIEndpoint != "https://gen3.example" || save { + t.Fatalf("unexpected helper credential: credential=%+v save=%v", cred, save) + } +} + +func TestNewRejectsProviderWithoutOperationalAdapter(t *testing.T) { + cfg := &config.Config{Remotes: map[config.Remote]config.RemoteSelect{ + "cgc": {Generic: &config.GenericRemote{Endpoint: "https://example.test", Provider: "cgc", Auth: "bearer"}}, + }} + if _, err := New(cfg, "cgc", drslog.GetLogger()); err == nil { + t.Fatal("expected unimplemented provider adapter to fail explicitly") + } +} + +func TestNewGenericGen3RejectsUnsupportedAuthentication(t *testing.T) { + for _, auth := range []string{"auto", "none", "basic", "google-adc", "provider-helper:other"} { + t.Run(auth, func(t *testing.T) { + cfg := &config.Config{Remotes: map[config.Remote]config.RemoteSelect{ + "gen3": {Generic: &config.GenericRemote{ + Endpoint: "https://gen3.example", Provider: "gen3", Auth: auth, + Scope: "program/project", Credential: "profile:stored", + }}, + }} + if _, err := New(cfg, "gen3", drslog.GetLogger()); err == nil { + t.Fatalf("expected Gen3 authentication method %q to fail explicitly", auth) + } + }) + } +} + func TestLocalClientResolvesBucketScopeMappings(t *testing.T) { setupTestRepo(t) @@ -114,3 +275,33 @@ func TestNewGitContextReadsLFSConcurrentTransfers(t *testing.T) { t.Fatalf("UploadConcurrency = %d, want 7", gitCtx.UploadConcurrency) } } + +func TestNewGitContextDiscoversBucketForScopeOnlyRemote(t *testing.T) { + setupTestRepo(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/data/buckets" { + t.Fatalf("request path = %q, want /data/buckets", r.URL.Path) + } + if got := r.Header.Get("Authorization"); got != "Bearer token" { + t.Fatalf("Authorization = %q, want bearer token", got) + } + _, _ = w.Write([]byte(`{"S3_BUCKETS":{"scope-bucket":{"programs":["/organization/org1/project/proj1"]}}}`)) + })) + defer server.Close() + + gitCtx, err := newGitContext(syconf.Credential{ + APIEndpoint: server.URL, + AccessToken: "token", + }, config.Gen3Remote{ + Endpoint: server.URL, + Organization: "org1", + ProjectID: "proj1", + }, drslog.GetLogger()) + if err != nil { + t.Fatalf("newGitContext failed: %v", err) + } + if gitCtx.BucketName != "scope-bucket" { + t.Fatalf("BucketName = %q, want scope-bucket", gitCtx.BucketName) + } +} diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go new file mode 100644 index 00000000..a856e0c5 --- /dev/null +++ b/internal/resolver/resolver.go @@ -0,0 +1,313 @@ +// Package resolver provides provider-neutral DRS metadata and access resolution. +package resolver + +import ( + "context" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" +) + +var ( + ErrCredentials = errors.New("google application default credentials are unavailable") + ErrUnauthorized = errors.New("not authorized to access AnVIL DRS object") + ErrNotFound = errors.New("AnVIL DRS object not found") +) + +type Resolver interface { + GetObject(context.Context, string) (*ResolvedObject, error) + GetAccess(context.Context, string, string) (*ResolvedAccess, error) +} + +// DownloadToCache resolves a fresh access URL, streams into a temporary file, +// and atomically promotes it. Neither the URL nor its headers are persisted. +func DownloadToCache(ctx context.Context, r Resolver, drsURI, destination string) error { + obj, err := r.GetObject(ctx, drsURI) + if err != nil { + return err + } + var access *ResolvedAccess + for _, method := range obj.AccessMethods { + // AnVIL objects commonly advertise a Google Storage (gs://) method + // before their HTTPS method. Select by the resolved URL scheme rather + // than the optional method type: older and test resolvers may omit type, + // while the actual URL is authoritative for the net/http downloader. + switch { + case method.AccessURL != nil && strings.TrimSpace(method.AccessURL.URL) != "": + if isHTTPAccessURL(method.AccessURL.URL) { + access = method.AccessURL + } + case strings.TrimSpace(method.AccessID) != "": + access, err = r.GetAccess(ctx, drsURI, method.AccessID) + if err != nil { + return err + } + if !isHTTPAccessURL(access.URL) { + access = nil + } + } + if access != nil { + break + } + } + if access == nil { + return fmt.Errorf("AnVIL DRS object has no supported HTTP(S) access method") + } + if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(destination), ".git-drs-download-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, access.URL, nil) + if err != nil { + tmp.Close() + return err + } + for _, header := range access.Headers { + key, value, ok := strings.Cut(header, ":") + key = strings.TrimSpace(key) + if !ok || key == "" { + tmp.Close() + return fmt.Errorf("AnVIL resolver returned an invalid access header") + } + req.Header.Add(key, strings.TrimSpace(value)) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + tmp.Close() + return fmt.Errorf("AnVIL data download failed") + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + tmp.Close() + return fmt.Errorf("AnVIL data download returned HTTP %d", resp.StatusCode) + } + hasher := sha256.New() + _, copyErr := io.Copy(io.MultiWriter(tmp, hasher), resp.Body) + closeErr := tmp.Close() + if copyErr != nil { + return fmt.Errorf("AnVIL data download interrupted: %w", copyErr) + } + if closeErr != nil { + return closeErr + } + if obj.Size >= 0 { + info, statErr := os.Stat(tmpName) + if statErr != nil { + return statErr + } + if info.Size() != obj.Size { + return fmt.Errorf("AnVIL data size mismatch: expected %d, got %d", obj.Size, info.Size()) + } + } + for _, checksum := range obj.Checksums { + checksumType := strings.ToLower(strings.TrimSpace(checksum.Type)) + if checksumType != "sha256" && checksumType != "sha-256" { + continue + } + expected := strings.TrimSpace(checksum.Checksum) + if prefix, value, ok := strings.Cut(expected, ":"); ok && + (strings.EqualFold(strings.TrimSpace(prefix), "sha256") || strings.EqualFold(strings.TrimSpace(prefix), "sha-256")) { + expected = strings.TrimSpace(value) + } + expectedBytes, decodeErr := hex.DecodeString(expected) + if decodeErr != nil || len(expectedBytes) != sha256.Size { + return fmt.Errorf("AnVIL DRS object has an invalid %s checksum", checksumType) + } + actual := hasher.Sum(nil) + if subtle.ConstantTimeCompare(actual, expectedBytes) != 1 { + return fmt.Errorf("AnVIL data checksum mismatch for %s", checksumType) + } + } + return os.Rename(tmpName, destination) +} + +func isHTTPAccessURL(raw string) bool { + u, err := url.Parse(strings.TrimSpace(raw)) + return err == nil && u.Host != "" && (u.Scheme == "http" || u.Scheme == "https") && u.User == nil +} + +type AccessMethod struct { + Type string `json:"type"` + AccessID string `json:"access_id,omitempty"` + AccessURL *ResolvedAccess `json:"access_url,omitempty"` +} + +type ResolvedObject struct { + DRSURI string + ID string `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + Checksums []Checksum `json:"checksums"` + AccessMethods []AccessMethod `json:"access_methods"` +} + +type Checksum struct { + Type string `json:"type"` + Checksum string `json:"checksum"` +} + +type ResolvedAccess struct { + URL string `json:"url"` + Headers []string `json:"headers"` +} + +// AnVILResolver routes every DRS authority through the configured trusted +// Terra/AnVIL resolver endpoint. Authorization is attached only by the HTTP +// client created for that endpoint; access URLs are returned ephemerally. +type AnVILResolver struct { + endpoint *url.URL + client *http.Client +} + +func NewAnVIL(ctx context.Context, endpoint string) (*AnVILResolver, error) { + u, err := trustedEndpoint(endpoint) + if err != nil { + return nil, err + } + creds, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/cloud-platform") + if err != nil { + return nil, fmt.Errorf("%w: run `gcloud auth application-default login`: %v", ErrCredentials, err) + } + return &AnVILResolver{endpoint: u, client: &http.Client{Transport: &oauth2.Transport{Base: http.DefaultTransport, Source: creds.TokenSource}}}, nil +} + +// NewAnVILWithClient supports contract tests and callers that already own an +// authenticated, refreshing client. +func NewAnVILWithClient(endpoint string, client *http.Client) (*AnVILResolver, error) { + u, err := trustedEndpoint(endpoint) + if err != nil { + return nil, err + } + if client == nil { + return nil, fmt.Errorf("HTTP client is required") + } + return &AnVILResolver{endpoint: u, client: client}, nil +} + +func trustedEndpoint(endpoint string) (*url.URL, error) { + u, err := url.Parse(strings.TrimSpace(endpoint)) + if err != nil || u.Host == "" || (u.Scheme != "https" && u.Scheme != "http") || u.User != nil { + return nil, fmt.Errorf("invalid AnVIL resolver endpoint") + } + return u, nil +} + +func NormalizeDRSURI(raw string) (string, string, error) { + trimmed := strings.TrimSpace(raw) + // DRS compact identifiers use a colon between the authority and object ID + // (for example, drs://drs.anv0:v2_...). net/url interprets that colon as + // the beginning of a port and rejects the URI before we can extract the ID. + if len(trimmed) >= len("drs://") && strings.EqualFold(trimmed[:len("drs://")], "drs://") { + remainder := trimmed[len("drs://"):] + if separator := strings.IndexByte(remainder, ':'); separator >= 0 && !strings.Contains(remainder[:separator], "/") { + authority, escapedID := remainder[:separator], remainder[separator+1:] + authorityURL, authorityErr := url.Parse("https://" + authority) + id, idErr := url.PathUnescape(escapedID) + if authorityErr != nil || authorityURL.Host != authority || authorityURL.Hostname() == "" || authorityURL.User != nil || + strings.ContainsAny(escapedID, "?#/") { + return "", "", fmt.Errorf("invalid DRS URI %q", raw) + } + if idErr != nil || strings.TrimSpace(id) == "" { + return "", "", fmt.Errorf("invalid DRS URI %q: object ID is required", raw) + } + return "drs://" + strings.ToLower(authority) + ":" + escapedID, id, nil + } + } + + u, err := url.Parse(trimmed) + if err != nil || !strings.EqualFold(u.Scheme, "drs") || u.Host == "" || u.RawQuery != "" || u.Fragment != "" { + return "", "", fmt.Errorf("invalid DRS URI %q", raw) + } + id, err := url.PathUnescape(strings.TrimPrefix(u.EscapedPath(), "/")) + if err != nil || strings.TrimSpace(id) == "" { + return "", "", fmt.Errorf("invalid DRS URI %q: object ID is required", raw) + } + u.Scheme = "drs" + u.Host = strings.ToLower(u.Host) + return u.String(), id, nil +} + +func (r *AnVILResolver) GetObject(ctx context.Context, drsURI string) (*ResolvedObject, error) { + canonical, id, err := NormalizeDRSURI(drsURI) + if err != nil { + return nil, err + } + var obj ResolvedObject + if err := r.getJSON(ctx, r.apiURL("objects", id), &obj); err != nil { + return nil, err + } + obj.DRSURI = canonical + if obj.ID == "" { + obj.ID = id + } + return &obj, nil +} + +func (r *AnVILResolver) GetAccess(ctx context.Context, drsURI, accessID string) (*ResolvedAccess, error) { + _, id, err := NormalizeDRSURI(drsURI) + if err != nil { + return nil, err + } + if strings.TrimSpace(accessID) == "" { + return nil, fmt.Errorf("access ID is required") + } + var access ResolvedAccess + if err := r.getJSON(ctx, r.apiURL("objects", id, "access", accessID), &access); err != nil { + return nil, err + } + if strings.TrimSpace(access.URL) == "" { + return nil, fmt.Errorf("AnVIL resolver returned an empty access URL") + } + return &access, nil +} + +func (r *AnVILResolver) apiURL(parts ...string) string { + u := *r.endpoint + segments := append([]string{u.Path, "ga4gh", "drs", "v1"}, parts...) + u.Path = path.Join(segments...) + return u.String() +} + +func (r *AnVILResolver) getJSON(ctx context.Context, endpoint string, dst any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return err + } + resp, err := r.client.Do(req) + if err != nil { + return fmt.Errorf("AnVIL resolver unavailable: %w", err) + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusUnauthorized, http.StatusForbidden: + return ErrUnauthorized + case http.StatusNotFound: + return ErrNotFound + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("AnVIL resolver returned HTTP %d", resp.StatusCode) + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(dst); err != nil { + return fmt.Errorf("decode AnVIL resolver response: %w", err) + } + return nil +} diff --git a/internal/resolver/resolver_test.go b/internal/resolver/resolver_test.go new file mode 100644 index 00000000..1b786d85 --- /dev/null +++ b/internal/resolver/resolver_test.go @@ -0,0 +1,285 @@ +package resolver + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestAnVILResolverContract(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/ga4gh/drs/v1/objects/object-1": + _, _ = w.Write([]byte(`{"id":"object-1","size":4,"checksums":[{"type":"sha256","checksum":"abcd"}],"access_methods":[{"type":"https","access_id":"a1"}]}`)) + case "/ga4gh/drs/v1/objects/object-1/access/a1": + _, _ = w.Write([]byte(`{"url":"https://storage.example/signed"}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + r, err := NewAnVILWithClient(server.URL, server.Client()) + if err != nil { + t.Fatal(err) + } + obj, err := r.GetObject(context.Background(), "DRS://AUTHORITY.EXAMPLE/object-1") + if err != nil { + t.Fatal(err) + } + if obj.DRSURI != "drs://authority.example/object-1" || obj.Size != 4 || obj.AccessMethods[0].AccessID != "a1" { + t.Fatalf("unexpected object: %+v", obj) + } + access, err := r.GetAccess(context.Background(), obj.DRSURI, "a1") + if err != nil || access.URL != "https://storage.example/signed" { + t.Fatalf("unexpected access result: %+v, %v", access, err) + } +} + +func TestDownloadToCacheUsesAccessURLHeaders(t *testing.T) { + const body = "data" + download := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Values("X-Provider-Token"); len(got) != 2 || got[0] != "first" || got[1] != "second:part" { + t.Errorf("unexpected provider headers: %q", got) + http.Error(w, "missing headers", http.StatusUnauthorized) + return + } + _, _ = w.Write([]byte(body)) + })) + defer download.Close() + + resolverServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/ga4gh/drs/v1/objects/object-1": + _, _ = w.Write([]byte(`{"id":"object-1","size":4,"access_methods":[{"type":"https","access_id":"a1"}]}`)) + case "/ga4gh/drs/v1/objects/object-1/access/a1": + _, _ = fmt.Fprintf(w, `{"url":%q,"headers":["X-Provider-Token: first","X-Provider-Token: second:part"]}`, download.URL) + default: + http.NotFound(w, r) + } + })) + defer resolverServer.Close() + + r, err := NewAnVILWithClient(resolverServer.URL, resolverServer.Client()) + if err != nil { + t.Fatal(err) + } + destination := filepath.Join(t.TempDir(), "cache", "object-1") + if err := DownloadToCache(context.Background(), r, "drs://example.org/object-1", destination); err != nil { + t.Fatalf("DownloadToCache returned error: %v", err) + } +} + +func TestDownloadToCacheUsesInlineAccessURL(t *testing.T) { + const body = "data" + download := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("X-Inline-Token"); got != "inline-secret" { + t.Errorf("unexpected inline access header: %q", got) + } + _, _ = w.Write([]byte(body)) + })) + defer download.Close() + + resolverServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ga4gh/drs/v1/objects/object-1" { + t.Errorf("inline access URL should not require an access endpoint request: %s", r.URL.Path) + http.NotFound(w, r) + return + } + _, _ = fmt.Fprintf(w, `{"id":"object-1","size":4,"access_methods":[{"type":"https","access_url":{"url":%q,"headers":["X-Inline-Token: inline-secret"]}}]}`, download.URL) + })) + defer resolverServer.Close() + + r, err := NewAnVILWithClient(resolverServer.URL, resolverServer.Client()) + if err != nil { + t.Fatal(err) + } + destination := filepath.Join(t.TempDir(), "cache", "object-1") + if err := DownloadToCache(context.Background(), r, "drs://example.org/object-1", destination); err != nil { + t.Fatalf("DownloadToCache returned error: %v", err) + } +} + +func TestDownloadToCacheSelectsUsableAccessMethodAfterFirst(t *testing.T) { + const body = "data" + download := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(body)) + })) + defer download.Close() + + resolverServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/ga4gh/drs/v1/objects/object-1": + _, _ = w.Write([]byte(`{"id":"object-1","size":4,"access_methods":[{"type":"gs","access_url":{"url":"gs://anvil-bucket/object-1"}},{"type":"https","access_id":"a2"}]}`)) + case "/ga4gh/drs/v1/objects/object-1/access/a2": + _, _ = fmt.Fprintf(w, `{"url":%q}`, download.URL) + default: + http.NotFound(w, r) + } + })) + defer resolverServer.Close() + + r, err := NewAnVILWithClient(resolverServer.URL, resolverServer.Client()) + if err != nil { + t.Fatal(err) + } + destination := filepath.Join(t.TempDir(), "cache", "object-1") + if err := DownloadToCache(context.Background(), r, "drs://example.org/object-1", destination); err != nil { + t.Fatalf("DownloadToCache returned error: %v", err) + } +} + +func TestDownloadToCacheRejectsNonHTTPAccessURLFromHTTPSMethod(t *testing.T) { + r := &staticResolver{object: &ResolvedObject{ + Size: 4, + AccessMethods: []AccessMethod{{ + Type: "https", + AccessURL: &ResolvedAccess{URL: "gs://anvil-bucket/object-1"}, + }}, + }} + err := DownloadToCache(context.Background(), r, "drs://example.org/object-1", filepath.Join(t.TempDir(), "cache", "object-1")) + if err == nil || !strings.Contains(err.Error(), "no supported HTTP(S) access method") { + t.Fatalf("expected unsupported access method error, got %v", err) + } +} + +func TestDownloadToCacheAcceptsHTTPAccessWithOmittedMethodType(t *testing.T) { + const body = "data" + download := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(body)) + })) + defer download.Close() + + r := &staticResolver{ + object: &ResolvedObject{ + Size: int64(len(body)), + AccessMethods: []AccessMethod{{AccessID: "access-1"}}, + }, + access: &ResolvedAccess{URL: download.URL}, + } + destination := filepath.Join(t.TempDir(), "cache", "object-1") + if err := DownloadToCache(context.Background(), r, "drs://example.org/object-1", destination); err != nil { + t.Fatalf("DownloadToCache returned error: %v", err) + } +} + +type staticResolver struct { + object *ResolvedObject + access *ResolvedAccess +} + +func (r *staticResolver) GetObject(context.Context, string) (*ResolvedObject, error) { + return r.object, nil +} + +func (r *staticResolver) GetAccess(context.Context, string, string) (*ResolvedAccess, error) { + if r.access != nil { + return r.access, nil + } + return nil, errors.New("unexpected GetAccess call") +} + +func TestDownloadToCacheValidatesSHA256BeforePromotion(t *testing.T) { + const body = "data" + correctSum := sha256.Sum256([]byte(body)) + incorrectSum := sha256.Sum256([]byte("oops")) // Same length as body. + + for _, test := range []struct { + name string + checksum string + wantErr bool + wantContent string + }{ + {name: "matching", checksum: hex.EncodeToString(correctSum[:]), wantContent: body}, + {name: "mismatching same-size payload", checksum: hex.EncodeToString(incorrectSum[:]), wantErr: true, wantContent: "previous"}, + } { + t.Run(test.name, func(t *testing.T) { + download := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(body)) + })) + defer download.Close() + + resolverServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ga4gh/drs/v1/objects/object-1" { + http.NotFound(w, r) + return + } + _, _ = fmt.Fprintf(w, `{"id":"object-1","size":4,"checksums":[{"type":"sha256","checksum":%q}],"access_methods":[{"type":"https","access_url":{"url":%q}}]}`, test.checksum, download.URL) + })) + defer resolverServer.Close() + + r, err := NewAnVILWithClient(resolverServer.URL, resolverServer.Client()) + if err != nil { + t.Fatal(err) + } + cacheDir := filepath.Join(t.TempDir(), "cache") + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + t.Fatal(err) + } + destination := filepath.Join(cacheDir, "object-1") + if err := os.WriteFile(destination, []byte("previous"), 0o644); err != nil { + t.Fatal(err) + } + + err = DownloadToCache(context.Background(), r, "drs://example.org/object-1", destination) + if test.wantErr && (err == nil || !strings.Contains(err.Error(), "checksum mismatch")) { + t.Fatalf("expected checksum mismatch, got %v", err) + } + if !test.wantErr && err != nil { + t.Fatalf("DownloadToCache returned error: %v", err) + } + content, readErr := os.ReadFile(destination) + if readErr != nil { + t.Fatal(readErr) + } + if string(content) != test.wantContent { + t.Fatalf("cached content = %q, want %q", content, test.wantContent) + } + if temporary, globErr := filepath.Glob(filepath.Join(cacheDir, ".git-drs-download-*")); globErr != nil || len(temporary) != 0 { + t.Fatalf("temporary downloads remain after validation: %v (glob error: %v)", temporary, globErr) + } + }) + } +} + +func TestAnVILResolverAcceptsCompactDRSURI(t *testing.T) { + const compactURI = "drs://drs.anv0:v2_e68887be-c583-375a-a773-48771192c8fa" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ga4gh/drs/v1/objects/v2_e68887be-c583-375a-a773-48771192c8fa" { + t.Fatalf("unexpected resolver path: %s", r.URL.Path) + } + _, _ = w.Write([]byte(`{"id":"v2_e68887be-c583-375a-a773-48771192c8fa","size":42}`)) + })) + defer server.Close() + + r, err := NewAnVILWithClient(server.URL, server.Client()) + if err != nil { + t.Fatal(err) + } + obj, err := r.GetObject(context.Background(), compactURI) + if err != nil { + t.Fatalf("compact DRS URI should be valid: %v", err) + } + if obj.DRSURI != compactURI || obj.ID != "v2_e68887be-c583-375a-a773-48771192c8fa" { + t.Fatalf("unexpected object: %+v", obj) + } +} + +func TestAnVILResolverRedactsErrorBodiesAndClassifiesAuthorization(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "Bearer secret signed=https://secret", http.StatusForbidden) + })) + defer server.Close() + r, _ := NewAnVILWithClient(server.URL, server.Client()) + _, err := r.GetObject(context.Background(), "drs://example.org/object") + if !errors.Is(err, ErrUnauthorized) || strings.Contains(err.Error(), "secret") { + t.Fatalf("expected redacted authorization error, got %v", err) + } +} diff --git a/internal/testutils/config.go b/internal/testutils/config.go index e8ab73bc..88a66f35 100644 --- a/internal/testutils/config.go +++ b/internal/testutils/config.go @@ -70,6 +70,15 @@ func CreateTestConfig(t *testing.T, tmpDir string, cfg *config.Config) { if remote.Gen3.Organization != "" { setConfig(prefix+".organization", remote.Gen3.Organization) } + } else if remote.Terra != nil { + setConfig(prefix+".type", "terra") + setConfig(prefix+".endpoint", remote.Terra.Endpoint) + if remote.Terra.Auth != "" { + setConfig(prefix+".auth", remote.Terra.Auth) + } + if remote.Terra.Mode != "" { + setConfig(prefix+".mode", remote.Terra.Mode) + } } else if remote.Local != nil { setConfig(prefix+".type", "local") setConfig(prefix+".endpoint", remote.Local.BaseURL) diff --git a/internal/transfer/download.go b/internal/transfer/download.go index 70928b5c..9bc95b2e 100644 --- a/internal/transfer/download.go +++ b/internal/transfer/download.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/calypr/git-drs/internal/drsobject" + "github.com/calypr/git-drs/internal/lfs" "github.com/calypr/git-drs/internal/lookup" "github.com/calypr/git-drs/internal/remoteruntime" drsapi "github.com/calypr/syfon/apigen/client/drs" @@ -42,7 +43,44 @@ func AccessURLForHashScope(ctx context.Context, drsCtx *remoteruntime.GitContext return &accessURL, &match, nil } +func AccessURLForDRSURI(ctx context.Context, drsCtx *remoteruntime.GitContext, drsURI string) (*drsapi.AccessURL, *drsapi.DrsObject, error) { + if strings.HasPrefix(drsURI, "//") { + drsURI = "drs:" + drsURI + } + obj, err := drsCtx.Client.DRS().GetObject(ctx, drsURI) + if err != nil { + return nil, nil, err + } + if obj.AccessMethods == nil || len(*obj.AccessMethods) == 0 { + return nil, nil, fmt.Errorf("no access methods available for DRS object %s", obj.Id) + } + accessType := (*obj.AccessMethods)[0].Type + if accessType == "" { + return nil, nil, fmt.Errorf("no access type found in access method for DRS object %s", obj.Id) + } + accessURL, err := drsCtx.Client.DRS().GetAccessURL(ctx, obj.Id, string(accessType)) + if err != nil { + return nil, nil, err + } + return &accessURL, &obj, nil +} + +func DownloadDRSURIToCachePath(ctx context.Context, drsCtx *remoteruntime.GitContext, drsURI, cachePath string) error { + if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { + return fmt.Errorf("mkdir for cache path: %w", err) + } + accessURL, obj, err := AccessURLForDRSURI(ctx, drsCtx, drsURI) + if err != nil { + return err + } + return downloadResolved(ctx, drsCtx, drsURI, cachePath, obj, accessURL) +} + func DownloadToCachePath(ctx context.Context, drsCtx *remoteruntime.GitContext, oid, cachePath string) error { + if lfs.IsDRSURI(oid) { + return DownloadDRSURIToCachePath(ctx, drsCtx, oid, cachePath) + } + if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { return fmt.Errorf("mkdir for cache path: %w", err) } diff --git a/internal/transfer/download_test.go b/internal/transfer/download_test.go index f88a5f8e..52d58755 100644 --- a/internal/transfer/download_test.go +++ b/internal/transfer/download_test.go @@ -20,6 +20,38 @@ type downloadRoundTripFunc func(*http.Request) (*http.Response, error) func (f downloadRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } +func TestDownloadToCachePathDispatchesDRSOIDToURIResolver(t *testing.T) { + t.Parallel() + + for _, oid := range []string{"drs://example.test/object-1", "//example.test/object-1"} { + oid := oid + t.Run(oid, func(t *testing.T) { + t.Parallel() + + var requestPath string + httpClient := &http.Client{Transport: downloadRoundTripFunc(func(r *http.Request) (*http.Response, error) { + requestPath = r.URL.Path + return nil, io.EOF + })} + raw, err := syclient.New("http://example.test", syclient.WithHTTPClient(httpClient)) + if err != nil { + t.Fatalf("syclient.New: %v", err) + } + + err = DownloadToCachePath(context.Background(), &remoteruntime.GitContext{Client: raw.(*syclient.Client)}, oid, filepath.Join(t.TempDir(), "object")) + if err == nil { + t.Fatal("DownloadToCachePath unexpectedly succeeded") + } + if strings.Contains(requestPath, "/checksum/") { + t.Fatalf("DRS OID was sent to checksum lookup: %s", requestPath) + } + if !strings.Contains(requestPath, "/objects/drs://example.test/object-1") { + t.Fatalf("DRS OID was not sent to URI lookup: %s", requestPath) + } + }) + } +} + func TestBulkAccessURLsForObjects(t *testing.T) { accessID := "s3" methods := []drsapi.AccessMethod{{Type: drsapi.AccessMethodTypeS3, AccessId: &accessID}} diff --git a/internal/transfer/push.go b/internal/transfer/push.go index 30c96b35..94c5b0fa 100644 --- a/internal/transfer/push.go +++ b/internal/transfer/push.go @@ -108,6 +108,14 @@ func (s *batchSyncSession) normalizeFiles(files map[string]lfs.LfsFileInfo) { if oid == "" { continue } + // DRS URI pointers identify an existing remote object; they are not + // SHA-256 checksums. Leave them to the URI-aware fetch path rather than + // sending the authority/object value through checksum lookup, + // registration, or upload synchronization. + if lfs.IsDRSURI(oid) { + s.debug("skipping DRS URI pointer during checksum push synchronization", "oid", oid) + continue + } if _, exists := s.filesByOID[oid]; exists { continue } diff --git a/internal/transfer/push_test.go b/internal/transfer/push_test.go index f64e6899..8a65afd7 100644 --- a/internal/transfer/push_test.go +++ b/internal/transfer/push_test.go @@ -39,6 +39,33 @@ func TestBatchSyncSessionNormalizeFilesDeduplicatesByOID(t *testing.T) { } } +func TestBatchSyncSessionNormalizeFilesExcludesDRSURIReferences(t *testing.T) { + session := &batchSyncSession{} + checksum := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + session.normalizeFiles(map[string]lfs.LfsFileInfo{ + "data/checksum.dat": { + Name: "data/checksum.dat", + Oid: checksum, + }, + "data/parsed-reference.dat": { + Name: "data/parsed-reference.dat", + Oid: "//authority.example/object-with-sha256", + }, + "data/canonical-reference.dat": { + Name: "data/canonical-reference.dat", + Oid: "drs://authority.example/another-object", + }, + }) + + if len(session.oids) != 1 || session.oids[0] != checksum { + t.Fatalf("expected only checksum OID in push synchronization, got %+v", session.oids) + } + if len(session.filesByOID) != 1 { + t.Fatalf("expected only checksum file in push synchronization, got %+v", session.filesByOID) + } +} + func TestAddURLObjectRegistersWithoutLocalPayloadUpload(t *testing.T) { t.Chdir(t.TempDir()) oid := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..62de6f0a --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,30 @@ +site_name: Git DRS Documentation +site_description: End-user documentation for Git-compatible DRS workflows +repo_url: https://github.com/calypr/git-drs +docs_dir: .pages-docs +site_dir: _site + +theme: + name: material + features: + - content.code.copy + - navigation.sections + - navigation.top + +markdown_extensions: + - admonition + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + +nav: + - Home: index.md + - Start here: + - Quick Start: quickstart.md + - Getting Started: getting-started.md + - Commands Reference: commands.md + - Work with data: + - Pointer Files and Reference State: pointer-files.md + - Add Existing S3 Objects: adding-s3-files.md + - Remove Files: remove-files.md + - Troubleshooting: troubleshooting.md diff --git a/scripts/anvil-add-ref-commands.sh b/scripts/anvil-add-ref-commands.sh new file mode 100755 index 00000000..f6aa91d3 --- /dev/null +++ b/scripts/anvil-add-ref-commands.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Generate git-drs add-ref commands from an AnVIL TSV export. + +Usage: + anvil-add-ref-commands.sh + +The input must contain columns named "files.drs_uri" and "files.file_name". +Commands are written to standard output; they are not executed. +EOF +} + +die() { + printf 'error: %s\n' "$*" >&2 + exit 1 +} + +split_tsv_line() { + local line=$1 + + # macOS ships Bash 3.2, which does not support namerefs. Return fields + # through this global array instead so the script works with both + # the system Bash on macOS and newer Bash releases. + tsv_fields=() + while [[ $line == *$'\t'* ]]; do + tsv_fields+=("${line%%$'\t'*}") + line=${line#*$'\t'} + done + tsv_fields+=("$line") +} + +if [[ ${1:-} == "-h" || ${1:-} == "--help" ]]; then + usage + exit 0 +fi + +[[ $# -eq 1 ]] || { + usage >&2 + exit 2 +} + +input=$1 +[[ -f $input ]] || die "TSV file not found: $input" +[[ -r $input ]] || die "TSV file is not readable: $input" + +IFS= read -r header_line < "$input" || die "TSV file is empty: $input" +header_line=${header_line%$'\r'} +header_line=${header_line#$'\xef\xbb\xbf'} + +declare -a headers +declare -a tsv_fields +split_tsv_line "$header_line" +headers=("${tsv_fields[@]}") + +drs_uri_index=-1 +file_name_index=-1 +for index in "${!headers[@]}"; do + case ${headers[$index]} in + files.drs_uri) drs_uri_index=$index ;; + files.file_name) file_name_index=$index ;; + esac +done + +((drs_uri_index >= 0)) || die 'required column not found: files.drs_uri' +((file_name_index >= 0)) || die 'required column not found: files.file_name' + +line_number=1 +while IFS= read -r line || [[ -n $line ]]; do + ((line_number += 1)) + line=${line%$'\r'} + [[ -n $line ]] || continue + + declare -a fields + split_tsv_line "$line" + fields=("${tsv_fields[@]}") + + drs_uri=${fields[$drs_uri_index]:-} + file_name=${fields[$file_name_index]:-} + [[ -n $drs_uri ]] || die "line $line_number has an empty files.drs_uri value" + [[ -n $file_name ]] || die "line $line_number has an empty files.file_name value" + + printf 'git drs add-ref --remote anvil %q %q\n' "$drs_uri" "$file_name" +done < <(tail -n +2 -- "$input")