fix(taskfile): coordinate cache cleanup with readers
This commit is contained in:
parent
b5aef1c287
commit
c46115b3ad
5 changed files with 168 additions and 5 deletions
1
go.mod
1
go.mod
|
|
@ -17,6 +17,7 @@ require (
|
|||
github.com/fsnotify/fsnotify v1.10.1
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0
|
||||
github.com/go-task/template v0.2.0
|
||||
github.com/gofrs/flock v0.12.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/hashicorp/go-getter v1.8.9
|
||||
github.com/joho/godotenv v1.5.1
|
||||
|
|
|
|||
2
go.sum
2
go.sum
|
|
@ -151,6 +151,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
|
|||
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
|
||||
github.com/go-task/template v0.2.0 h1:xW7ek0o65FUSTbKcSNeg2Vyf/I7wYXFgLUznptvviBE=
|
||||
github.com/go-task/template v0.2.0/go.mod h1:dbdoUb6qKnHQi1y6o+IdIrs0J4o/SEhSTA6bbzZmdtc=
|
||||
github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E=
|
||||
github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
|
|
|
|||
|
|
@ -9,8 +9,10 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
giturls "github.com/chainguard-dev/git-urls"
|
||||
"github.com/gofrs/flock"
|
||||
"github.com/hashicorp/go-getter"
|
||||
|
||||
"github.com/go-task/task/v3/errors"
|
||||
|
|
@ -50,6 +52,8 @@ var globalGitRepoCache = &gitRepoCache{
|
|||
|
||||
const gitCacheDirName = "task-git-repos"
|
||||
|
||||
const gitCacheLockRetryDelay = 10 * time.Millisecond
|
||||
|
||||
var (
|
||||
fallbackGitCacheOnce sync.Once
|
||||
fallbackGitCacheDir string
|
||||
|
|
@ -88,18 +92,56 @@ func prepareGitCacheRoot(root string) error {
|
|||
}
|
||||
|
||||
func CleanGitCache() error {
|
||||
root, err := gitCacheRoot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return cleanGitCache(root)
|
||||
}
|
||||
|
||||
func cleanGitCache(root string) error {
|
||||
unlock, err := lockGitCache(context.Background(), root, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = unlock() }()
|
||||
|
||||
// Clear the in-memory locks map to prevent memory leak
|
||||
globalGitRepoCache.mu.Lock()
|
||||
globalGitRepoCache.locks = make(map[string]*sync.Mutex)
|
||||
globalGitRepoCache.mu.Unlock()
|
||||
|
||||
root, err := gitCacheRoot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.RemoveAll(root)
|
||||
}
|
||||
|
||||
// lockGitCache coordinates readers with cleanup across Task processes. A
|
||||
// reader holds a shared lock until it has read every remote Taskfile; cleanup
|
||||
// takes an exclusive lock so one process cannot remove another process's clone.
|
||||
func lockGitCache(ctx context.Context, root string, shared bool) (func() error, error) {
|
||||
lockDir := filepath.Dir(root)
|
||||
if err := prepareGitCacheRoot(lockDir); err != nil {
|
||||
return nil, fmt.Errorf("failed to prepare git cache lock directory: %w", err)
|
||||
}
|
||||
lock := flock.New(root+".lock",
|
||||
flock.SetPermissions(0o600),
|
||||
flock.SetFlag(os.O_CREATE|os.O_RDWR),
|
||||
)
|
||||
var locked bool
|
||||
var err error
|
||||
if shared {
|
||||
locked, err = lock.TryRLockContext(ctx, gitCacheLockRetryDelay)
|
||||
} else {
|
||||
locked, err = lock.TryLockContext(ctx, gitCacheLockRetryDelay)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to lock git cache: %w", err)
|
||||
}
|
||||
if !locked {
|
||||
return nil, fmt.Errorf("failed to lock git cache")
|
||||
}
|
||||
return lock.Unlock, nil
|
||||
}
|
||||
|
||||
func NewGitNode(
|
||||
entrypoint string,
|
||||
dir string,
|
||||
|
|
|
|||
|
|
@ -2,11 +2,14 @@ package taskfile
|
|||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
|
@ -383,3 +386,103 @@ func TestGetOrCloneRepo_ReclonesWithoutGitDir(t *testing.T) { //nolint:parallelt
|
|||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "failed to clone repository")
|
||||
}
|
||||
|
||||
type blockingGitCacheReadNode struct {
|
||||
Node
|
||||
readyPath string
|
||||
releasePath string
|
||||
}
|
||||
|
||||
func (node blockingGitCacheReadNode) Read() ([]byte, error) {
|
||||
if err := os.WriteFile(node.readyPath, nil, 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := os.Stat(node.releasePath); err == nil {
|
||||
return node.Node.Read()
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
return nil, os.ErrDeadlineExceeded
|
||||
}
|
||||
|
||||
func TestGitCacheReaderHelper(t *testing.T) {
|
||||
t.Parallel()
|
||||
if os.Getenv("GO_TASK_GIT_CACHE_READER_HELPER") != "1" {
|
||||
return
|
||||
}
|
||||
|
||||
baseNode, err := NewFileNode(os.Getenv("GO_TASK_GIT_CACHE_TASKFILE"), "")
|
||||
require.NoError(t, err)
|
||||
node := blockingGitCacheReadNode{
|
||||
Node: baseNode,
|
||||
readyPath: os.Getenv("GO_TASK_GIT_CACHE_READY"),
|
||||
releasePath: os.Getenv("GO_TASK_GIT_CACHE_RELEASE"),
|
||||
}
|
||||
_, err = NewReader().Read(context.Background(), node)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestReaderReadPreventsConcurrentGitCacheCleanup(t *testing.T) { //nolint:paralleltest // t.Setenv changes the user cache directory
|
||||
cacheDir := t.TempDir()
|
||||
setUserCacheDir(t, cacheDir)
|
||||
root, err := gitCacheRoot()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.MkdirAll(root, 0o700))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(root, "cached-repo"), nil, 0o600))
|
||||
|
||||
taskfile := filepath.Join(t.TempDir(), "Taskfile.yml")
|
||||
require.NoError(t, os.WriteFile(taskfile, []byte("version: '3'\n"), 0o600))
|
||||
readyPath := filepath.Join(t.TempDir(), "reader-ready")
|
||||
releasePath := filepath.Join(filepath.Dir(readyPath), "release-reader")
|
||||
|
||||
//nolint:gosec // Re-execute this test binary to model a separate Task process.
|
||||
cmd := exec.CommandContext(context.Background(), os.Args[0], "-test.run=^TestGitCacheReaderHelper$")
|
||||
cmd.Env = append(os.Environ(),
|
||||
"GO_TASK_GIT_CACHE_READER_HELPER=1",
|
||||
"GO_TASK_GIT_CACHE_TASKFILE="+taskfile,
|
||||
"GO_TASK_GIT_CACHE_READY="+readyPath,
|
||||
"GO_TASK_GIT_CACHE_RELEASE="+releasePath,
|
||||
)
|
||||
cmd.Stdout = io.Discard
|
||||
cmd.Stderr = io.Discard
|
||||
require.NoError(t, cmd.Start())
|
||||
t.Cleanup(func() {
|
||||
if cmd.ProcessState == nil {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
}
|
||||
})
|
||||
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := os.Stat(readyPath); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if _, err := os.Stat(readyPath); err != nil {
|
||||
t.Fatalf("reader helper did not start reading: %s", err)
|
||||
}
|
||||
|
||||
cleanupDone := make(chan error, 1)
|
||||
go func() { cleanupDone <- cleanGitCache(root) }()
|
||||
select {
|
||||
case err := <-cleanupDone:
|
||||
_ = os.WriteFile(releasePath, nil, 0o600)
|
||||
t.Fatalf("git cache cleanup completed while another process was reading: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
require.NoError(t, os.WriteFile(releasePath, nil, 0o600))
|
||||
select {
|
||||
case err := <-cleanupDone:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("git cache cleanup did not resume after the reader finished")
|
||||
}
|
||||
require.NoError(t, cmd.Wait())
|
||||
if _, err := os.Stat(root); !os.IsNotExist(err) {
|
||||
t.Fatalf("expected git cache root to be removed, stat error: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,9 +90,24 @@ func (r *Reader) Options(opts ...ReaderOption) {
|
|||
// building an [ast.TaskfileGraph] as it goes. If any errors occur, they will be
|
||||
// returned immediately.
|
||||
func (r *Reader) Read(ctx context.Context, node Node) (*ast.TaskfileGraph, error) {
|
||||
root, err := gitCacheRoot()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The Task timeout applies to remote I/O, not waiting for local cache
|
||||
// cleanup; local Taskfiles must still be readable when the timeout is zero.
|
||||
unlock, err := lockGitCache(context.Background(), root, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to lock git cache while reading Taskfiles: %w", err)
|
||||
}
|
||||
|
||||
// Clean up git cache after reading all taskfiles
|
||||
defer func() {
|
||||
if err := CleanGitCache(); err != nil {
|
||||
if err := unlock(); err != nil {
|
||||
r.debugf("failed to unlock git cache: %s\n", err.Error())
|
||||
return
|
||||
}
|
||||
if err := cleanGitCache(root); err != nil {
|
||||
r.debugf("failed to clean git cache: %s\n", err.Error())
|
||||
}
|
||||
}()
|
||||
|
|
|
|||
Loading…
Reference in a new issue