From 2ce12745a6f85928f12535527b8a09d47c64bf5c Mon Sep 17 00:00:00 2001 From: Caj Larsson Date: Tue, 3 May 2022 22:11:28 +0800 Subject: [PATCH] swampfile.Repository.DeleteOlderThan --- dataswamp/services.go | 33 ++++++++++++++--- dataswamp/services_test.go | 34 ++++++++++++++--- dataswamp/swampfile/file_contract.go | 2 +- dataswamp/swampfile/repository.go | 6 +-- dataswamp/swampfile/valueobjects.go | 5 +++ infrastructure/fs/swampfile/repository.go | 22 ++++++----- .../fs/swampfile/repository_test.go | 8 +--- infrastructure/memory/swampfile/repository.go | 37 ++++++++++--------- 8 files changed, 98 insertions(+), 49 deletions(-) diff --git a/dataswamp/services.go b/dataswamp/services.go index f921f84..ec89fa3 100644 --- a/dataswamp/services.go +++ b/dataswamp/services.go @@ -4,11 +4,10 @@ import ( "caj-larsson/bog/dataswamp/namespace" "caj-larsson/bog/dataswamp/swampfile" "io" - "path" - "path/filepath" "strconv" - "strings" "time" + // "errors" + // "fmt" ) type SwampFileService struct { @@ -82,7 +81,7 @@ func (s SwampFileService) SaveFile(ref swampfile.FileReference, src io.Reader, s var buf = make([]byte, 1) - overread, err :=src.Read(buf) + overread, err := src.Read(buf) if overread > 0 || err != io.EOF { s.swamp_file_repo.Delete(ref.Path, strconv.FormatInt(ns.ID, 10)) @@ -114,6 +113,28 @@ func (s SwampFileService) OpenOutFile(ref swampfile.FileReference) (swampfile.Sw return f, nil } -func CleanPath(inpath string) string { - return filepath.FromSlash(path.Clean("/" + strings.Trim(inpath, "/"))) +func (s SwampFileService) CleanUpExpiredFiles() error { + nss, err := s.namespace_repo.All() + + if err != nil { + return err + } + + for _, ns := range nss { + expiry := time.Now().Add(-ns.AllowanceDuration) + dfs, err := s.swamp_file_repo.DeleteOlderThan(strconv.FormatInt(ns.ID, 10), expiry) + + // panic(errors.New(fmt.Sprintf("%+v", dfs))) + for _, df := range dfs { + + ns.FileQuota.Remove(df.Size) + } + + if err != nil { + panic(err) + } + + s.namespace_repo.Update(ns.ID, ns) + } + return nil } diff --git a/dataswamp/services_test.go b/dataswamp/services_test.go index c6a8d34..c92bd6f 100644 --- a/dataswamp/services_test.go +++ b/dataswamp/services_test.go @@ -4,6 +4,7 @@ import ( "bytes" "caj-larsson/bog/dataswamp/swampfile" "github.com/matryer/is" + "github.com/spf13/afero" "testing" "time" // "caj-larsson/bog/dataswamp/namespace" @@ -11,7 +12,7 @@ import ( m_swampfile "caj-larsson/bog/infrastructure/memory/swampfile" ) -var file_ref1 = swampfile.FileReference{"/ptah1", "ns1"} +var file_ref1 = swampfile.FileReference{"/path1", "ns1"} var file_ref2 = swampfile.FileReference{"/path1", "ns2"} var file_ref3 = swampfile.FileReference{"/path2", "ns1"} @@ -85,14 +86,13 @@ func TestNSIsolation(t *testing.T) { is.Equal(outfile.String(), "My bog data ns1") } - func TestPathStrictMode(t *testing.T) { is := is.New(t) s := NewTestSwampFileService() ns_file := bytes.NewBufferString("My bog data ns1") - ref := swampfile.FileReference { + ref := swampfile.FileReference{ "/path/../with/../backrefs", "ns1", } @@ -106,7 +106,6 @@ func TestPathStrictMode(t *testing.T) { is.Equal(err, swampfile.ErrUnacceptablePath) } - func TestQuotaWithContenSizeLieOver(t *testing.T) { is := is.New(t) s := NewTestSwampFileService() @@ -122,7 +121,6 @@ func TestQuotaWithContenSizeLieOver(t *testing.T) { is.Equal(err, swampfile.ErrContentSizeExceeded) } - func TestQuotaWithContenSizeLieUnder(t *testing.T) { is := is.New(t) s := NewTestSwampFileService() @@ -133,3 +131,29 @@ func TestQuotaWithContenSizeLieUnder(t *testing.T) { is.Equal(err, swampfile.ErrContentSizeExaggerated) } + +func TestCleanUpExpired(t *testing.T) { + is := is.New(t) + + fs := afero.NewMemMapFs() + file_repo := m_swampfile.Repository{fs} + ns_repo := m_namespace.NewRepository() + s := NewSwampFileService(ns_repo, file_repo, 1024, time.Hour) + + fakefile := bytes.NewBufferString("My bog data") + err := s.SaveFile(file_ref1, fakefile, int64(fakefile.Len())) + is.NoErr(err) + + fakefile = bytes.NewBufferString("My bog data") + err = s.SaveFile(file_ref3, fakefile, int64(fakefile.Len())) + is.NoErr(err) + + err = fs.Chtimes("1/path1", time.Now(), time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)) + is.NoErr(err) + is.NoErr(s.CleanUpExpiredFiles()) + + ns, err := ns_repo.GetByName("ns1") + is.NoErr(err) + + is.Equal(ns.FileQuota.CurrentUsage, int64(len("My bog data"))) +} diff --git a/dataswamp/swampfile/file_contract.go b/dataswamp/swampfile/file_contract.go index c0b4e45..4e80593 100644 --- a/dataswamp/swampfile/file_contract.go +++ b/dataswamp/swampfile/file_contract.go @@ -57,7 +57,7 @@ func basicFileOperationContract(fac func() Repository, t *testing.T) { expiring_file.Write([]byte(testdata)) expiring_file.Close() - err = repo.DeleteOlderThan("ua1", time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)) + _, err = repo.DeleteOlderThan("ua1", time.Now().Add(time.Hour)) is.NoErr(err) expired_file, err := repo.Open("deep/dir/expiring.new", "ua1") diff --git a/dataswamp/swampfile/repository.go b/dataswamp/swampfile/repository.go index 620b89b..12fe60f 100644 --- a/dataswamp/swampfile/repository.go +++ b/dataswamp/swampfile/repository.go @@ -4,7 +4,7 @@ import "time" type Repository interface { Create(filename string, namespace_stub string) (SwampInFile, error) - Open(filename string, namespace_stub string) (SwampOutFile, error) - Delete(filename string, namespace_stub string) - DeleteOlderThan(namespace_stub string, ts time.Time) error + Open(filename string, namespace_stub string) (SwampOutFile, error) + Delete(filename string, namespace_stub string) + DeleteOlderThan(namespace_stub string, ts time.Time) ([]DeletedSwampFile, error) } diff --git a/dataswamp/swampfile/valueobjects.go b/dataswamp/swampfile/valueobjects.go index 8f74da9..9bec9c2 100644 --- a/dataswamp/swampfile/valueobjects.go +++ b/dataswamp/swampfile/valueobjects.go @@ -31,6 +31,11 @@ type FileReference struct { UserAgent string } +type DeletedSwampFile struct { + Path string + Size int64 +} + func (fr *FileReference) Clean(strict bool) error { c := filepath.FromSlash(path.Clean("/" + strings.Trim(fr.Path, "/"))) diff --git a/infrastructure/fs/swampfile/repository.go b/infrastructure/fs/swampfile/repository.go index 5ed6766..5e42941 100644 --- a/infrastructure/fs/swampfile/repository.go +++ b/infrastructure/fs/swampfile/repository.go @@ -2,12 +2,12 @@ package swampfile import ( "caj-larsson/bog/dataswamp/swampfile" + "errors" + "io/fs" "os" "path" "path/filepath" - "io/fs" "time" - "errors" ) var ErrDirtyRepo = errors.New("Dirty repository without flag") @@ -44,7 +44,8 @@ func (f FileSystemSwampFileData) Size() int64 { } func (f FileSystemSwampFileData) Modified() time.Time { - return time.Now() + stat, _ := f.file.Stat() + return stat.ModTime() } type Repository struct { @@ -62,7 +63,7 @@ func NewRepository(root string) (*Repository, error) { if len(fi) == 0 { // New Repository, write flagfile - f, err := os.OpenFile(flagpath, os.O_CREATE, 0644) + f, err := os.OpenFile(flagpath, os.O_CREATE, 0644) if err != nil { return nil, err } @@ -75,10 +76,9 @@ func NewRepository(root string) (*Repository, error) { return nil, ErrDirtyRepo } flagfile.Close() - return &Repository { root }, nil + return &Repository{root}, nil } - func (f Repository) absPath(filename string, namespace_ns string) string { return path.Join(f.Root, namespace_ns, filename) } @@ -125,7 +125,9 @@ func (f Repository) Delete(filename string, namespace_ns string) { os.Remove(abs_path) } -func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) error { +func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) ([]swampfile.DeletedSwampFile, error) { + df := []swampfile.DeletedSwampFile{} + dr := path.Join(r.Root, namespace_stub) err := filepath.Walk(dr, func(path string, info fs.FileInfo, err error) error { if err != nil { @@ -134,8 +136,8 @@ func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) error { mode := info.Mode() if mode.IsRegular() { - if ts.Before(info.ModTime()) { - + if info.ModTime().Before(ts) { + df = append(df, swampfile.DeletedSwampFile{path, info.Size()}) err = os.Remove(path) if err != nil { panic(err) @@ -150,5 +152,5 @@ func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) error { return nil }) - return err + return df, err } diff --git a/infrastructure/fs/swampfile/repository_test.go b/infrastructure/fs/swampfile/repository_test.go index 6c1102c..55af122 100644 --- a/infrastructure/fs/swampfile/repository_test.go +++ b/infrastructure/fs/swampfile/repository_test.go @@ -2,10 +2,10 @@ package swampfile import ( "caj-larsson/bog/dataswamp/swampfile" - "path" + "github.com/matryer/is" "os" + "path" "testing" - "github.com/matryer/is" ) func newRepoDir(t *testing.T) string { @@ -19,7 +19,6 @@ func newRepoDir(t *testing.T) string { return d } - func TestFsFileRepo(t *testing.T) { var fac = func() swampfile.Repository { @@ -34,7 +33,6 @@ func TestFsFileRepo(t *testing.T) { swampfile.RepositoryContract(fac, t) } - func TestEmptyDir(t *testing.T) { is := is.New(t) dir_path := newRepoDir(t) @@ -46,7 +44,6 @@ func TestEmptyDir(t *testing.T) { is.NoErr(err) } - func TestDirtyDir(t *testing.T) { is := is.New(t) dir_path := newRepoDir(t) @@ -59,7 +56,6 @@ func TestDirtyDir(t *testing.T) { is.Equal(err, ErrDirtyRepo) } - func TestDirtyWithFlag(t *testing.T) { is := is.New(t) dir_path := newRepoDir(t) diff --git a/infrastructure/memory/swampfile/repository.go b/infrastructure/memory/swampfile/repository.go index 73f7bc3..0479a36 100644 --- a/infrastructure/memory/swampfile/repository.go +++ b/infrastructure/memory/swampfile/repository.go @@ -1,16 +1,14 @@ package swampfile import ( + "caj-larsson/bog/dataswamp/swampfile" + "github.com/spf13/afero" + "io/fs" "os" "path" "time" - "io/fs" - "caj-larsson/bog/dataswamp/swampfile" - "github.com/spf13/afero" ) - - type SwampFile struct { filename string file afero.File @@ -48,7 +46,7 @@ func (f SwampFile) Modified() time.Time { // The actual repository type Repository struct { - fs afero.Fs + Fs afero.Fs } func NewRepository() swampfile.Repository { @@ -58,8 +56,8 @@ func NewRepository() swampfile.Repository { func (r Repository) Create(filename string, namespace_stub string) (swampfile.SwampInFile, error) { abs_path := path.Join(namespace_stub, filename) dir := path.Dir(abs_path) - r.fs.MkdirAll(dir, 0750) - file, err := r.fs.OpenFile(abs_path, os.O_RDWR|os.O_CREATE, 0644) + r.Fs.MkdirAll(dir, 0750) + file, err := r.Fs.OpenFile(abs_path, os.O_RDWR|os.O_CREATE, 0644) if err != nil { panic(err) @@ -74,8 +72,8 @@ func (r Repository) Open(filename string, namespace_stub string) (swampfile.Swam abs_path := path.Join(namespace_stub, filename) dir := path.Dir(abs_path) - r.fs.MkdirAll(dir, 0750) - file, err := r.fs.OpenFile(abs_path, os.O_RDONLY, 0644) + r.Fs.MkdirAll(dir, 0750) + file, err := r.Fs.OpenFile(abs_path, os.O_RDONLY, 0644) if err != nil { return nil, swampfile.ErrNotExists @@ -87,26 +85,29 @@ func (r Repository) Open(filename string, namespace_stub string) (swampfile.Swam func (r Repository) Delete(filename string, namespace_stub string) { abs_path := path.Join(namespace_stub, filename) - r.fs.Remove(abs_path) + r.Fs.Remove(abs_path) } -func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) error { - err := afero.Walk(r.fs, namespace_stub, func(path string, info fs.FileInfo, err error) error { +func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) ([]swampfile.DeletedSwampFile, error) { + df := []swampfile.DeletedSwampFile{} + + err := afero.Walk(r.Fs, namespace_stub, func(path string, info fs.FileInfo, err error) error { if err != nil { return err } - if info.Mode().IsRegular() && ts.Before(info.ModTime()) { - r.fs.Remove(path) + if info.Mode().IsRegular() { + if info.ModTime().Before(ts) { + df = append(df, swampfile.DeletedSwampFile{path, info.Size()}) + r.Fs.Remove(path) + } return nil } if !info.Mode().IsDir() { return swampfile.ErrCorrupted } - return nil }) - - return err + return df, err }