swampfile.Repository.DeleteOlderThan

dataswamp/services.go

@@ -4,11 +4,10 @@ import (
 	"caj-larsson/bog/dataswamp/namespace"
 	"caj-larsson/bog/dataswamp/swampfile"
 	"io"
-	"path"
-	"path/filepath"
 	"strconv"
-	"strings"
 	"time"
+	//	"errors"
+	//	"fmt"
 )
 
 type SwampFileService struct {
@@ -82,7 +81,7 @@ func (s SwampFileService) SaveFile(ref swampfile.FileReference, src io.Reader, s
 
 	var buf = make([]byte, 1)
 
-	overread, err :=src.Read(buf)
+	overread, err := src.Read(buf)
 
 	if overread > 0 || err != io.EOF {
 		s.swamp_file_repo.Delete(ref.Path, strconv.FormatInt(ns.ID, 10))
@@ -114,6 +113,28 @@ func (s SwampFileService) OpenOutFile(ref swampfile.FileReference) (swampfile.Sw
 	return f, nil
 }
 
-func CleanPath(inpath string) string {
+func (s SwampFileService) CleanUpExpiredFiles() error {
-	return filepath.FromSlash(path.Clean("/" + strings.Trim(inpath, "/")))
+	nss, err := s.namespace_repo.All()
+
+	if err != nil {
+		return err
+	}
+
+	for _, ns := range nss {
+		expiry := time.Now().Add(-ns.AllowanceDuration)
+		dfs, err := s.swamp_file_repo.DeleteOlderThan(strconv.FormatInt(ns.ID, 10), expiry)
+
+		// panic(errors.New(fmt.Sprintf("%+v", dfs)))
+		for _, df := range dfs {
+
+			ns.FileQuota.Remove(df.Size)
+		}
+
+		if err != nil {
+			panic(err)
+		}
+
+		s.namespace_repo.Update(ns.ID, ns)
+	}
+	return nil
 }

dataswamp/services_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"caj-larsson/bog/dataswamp/swampfile"
 	"github.com/matryer/is"
+	"github.com/spf13/afero"
 	"testing"
 	"time"
 	//	"caj-larsson/bog/dataswamp/namespace"
@@ -11,7 +12,7 @@ import (
 	m_swampfile "caj-larsson/bog/infrastructure/memory/swampfile"
 )
 
-var file_ref1 = swampfile.FileReference{"/ptah1", "ns1"}
+var file_ref1 = swampfile.FileReference{"/path1", "ns1"}
 var file_ref2 = swampfile.FileReference{"/path1", "ns2"}
 var file_ref3 = swampfile.FileReference{"/path2", "ns1"}
 
@@ -85,14 +86,13 @@ func TestNSIsolation(t *testing.T) {
 	is.Equal(outfile.String(), "My bog data ns1")
 }
 
-
 func TestPathStrictMode(t *testing.T) {
 	is := is.New(t)
 	s := NewTestSwampFileService()
 
 	ns_file := bytes.NewBufferString("My bog data ns1")
 
-	ref := swampfile.FileReference {
+	ref := swampfile.FileReference{
 		"/path/../with/../backrefs",
 		"ns1",
 	}
@@ -106,7 +106,6 @@ func TestPathStrictMode(t *testing.T) {
 	is.Equal(err, swampfile.ErrUnacceptablePath)
 }
 
-
 func TestQuotaWithContenSizeLieOver(t *testing.T) {
 	is := is.New(t)
 	s := NewTestSwampFileService()
@@ -122,7 +121,6 @@ func TestQuotaWithContenSizeLieOver(t *testing.T) {
 	is.Equal(err, swampfile.ErrContentSizeExceeded)
 }
 
-
 func TestQuotaWithContenSizeLieUnder(t *testing.T) {
 	is := is.New(t)
 	s := NewTestSwampFileService()
@@ -133,3 +131,29 @@ func TestQuotaWithContenSizeLieUnder(t *testing.T) {
 
 	is.Equal(err, swampfile.ErrContentSizeExaggerated)
 }
+
+func TestCleanUpExpired(t *testing.T) {
+	is := is.New(t)
+
+	fs := afero.NewMemMapFs()
+	file_repo := m_swampfile.Repository{fs}
+	ns_repo := m_namespace.NewRepository()
+	s := NewSwampFileService(ns_repo, file_repo, 1024, time.Hour)
+
+	fakefile := bytes.NewBufferString("My bog data")
+	err := s.SaveFile(file_ref1, fakefile, int64(fakefile.Len()))
+	is.NoErr(err)
+
+	fakefile = bytes.NewBufferString("My bog data")
+	err = s.SaveFile(file_ref3, fakefile, int64(fakefile.Len()))
+	is.NoErr(err)
+
+	err = fs.Chtimes("1/path1", time.Now(), time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))
+	is.NoErr(err)
+	is.NoErr(s.CleanUpExpiredFiles())
+
+	ns, err := ns_repo.GetByName("ns1")
+	is.NoErr(err)
+
+	is.Equal(ns.FileQuota.CurrentUsage, int64(len("My bog data")))
+}

dataswamp/swampfile/file_contract.go

@@ -57,7 +57,7 @@ func basicFileOperationContract(fac func() Repository, t *testing.T) {
 	expiring_file.Write([]byte(testdata))
 	expiring_file.Close()
 
-	err = repo.DeleteOlderThan("ua1", time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))
+	_, err = repo.DeleteOlderThan("ua1", time.Now().Add(time.Hour))
 	is.NoErr(err)
 
 	expired_file, err := repo.Open("deep/dir/expiring.new", "ua1")

dataswamp/swampfile/repository.go

@@ -4,7 +4,7 @@ import "time"
 
 type Repository interface {
 	Create(filename string, namespace_stub string) (SwampInFile, error)
-	Open(filename string, namespace_stub  string) (SwampOutFile, error)
+	Open(filename string, namespace_stub string) (SwampOutFile, error)
-	Delete(filename string, namespace_stub  string)
+	Delete(filename string, namespace_stub string)
-	DeleteOlderThan(namespace_stub string, ts time.Time) error
+	DeleteOlderThan(namespace_stub string, ts time.Time) ([]DeletedSwampFile, error)
 }

dataswamp/swampfile/valueobjects.go

@@ -31,6 +31,11 @@ type FileReference struct {
 	UserAgent string
 }
 
+type DeletedSwampFile struct {
+	Path string
+	Size int64
+}
+
 func (fr *FileReference) Clean(strict bool) error {
 	c := filepath.FromSlash(path.Clean("/" + strings.Trim(fr.Path, "/")))
 

infrastructure/fs/swampfile/repository.go

@@ -2,12 +2,12 @@ package swampfile
 
 import (
 	"caj-larsson/bog/dataswamp/swampfile"
+	"errors"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
-	"io/fs"
 	"time"
-	"errors"
 )
 
 var ErrDirtyRepo = errors.New("Dirty repository without flag")
@@ -44,7 +44,8 @@ func (f FileSystemSwampFileData) Size() int64 {
 }
 
 func (f FileSystemSwampFileData) Modified() time.Time {
-	return time.Now()
+	stat, _ := f.file.Stat()
+	return stat.ModTime()
 }
 
 type Repository struct {
@@ -62,7 +63,7 @@ func NewRepository(root string) (*Repository, error) {
 
 	if len(fi) == 0 {
 		// New Repository, write flagfile
-			f, err := os.OpenFile(flagpath, os.O_CREATE, 0644)
+		f, err := os.OpenFile(flagpath, os.O_CREATE, 0644)
 		if err != nil {
 			return nil, err
 		}
@@ -75,10 +76,9 @@ func NewRepository(root string) (*Repository, error) {
 		return nil, ErrDirtyRepo
 	}
 	flagfile.Close()
-	return &Repository { root }, nil
+	return &Repository{root}, nil
 }
 
-
 func (f Repository) absPath(filename string, namespace_ns string) string {
 	return path.Join(f.Root, namespace_ns, filename)
 }
@@ -125,7 +125,9 @@ func (f Repository) Delete(filename string, namespace_ns string) {
 	os.Remove(abs_path)
 }
 
-func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) error {
+func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) ([]swampfile.DeletedSwampFile, error) {
+	df := []swampfile.DeletedSwampFile{}
+
 	dr := path.Join(r.Root, namespace_stub)
 	err := filepath.Walk(dr, func(path string, info fs.FileInfo, err error) error {
 		if err != nil {
@@ -134,8 +136,8 @@ func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) error {
 		mode := info.Mode()
 
 		if mode.IsRegular() {
-			if ts.Before(info.ModTime()) {
+			if info.ModTime().Before(ts) {
-
+				df = append(df, swampfile.DeletedSwampFile{path, info.Size()})
 				err = os.Remove(path)
 				if err != nil {
 					panic(err)
@@ -150,5 +152,5 @@ func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) error {
 
 		return nil
 	})
-	return err
+	return df, err
 }

infrastructure/fs/swampfile/repository_test.go

@@ -2,10 +2,10 @@ package swampfile
 
 import (
 	"caj-larsson/bog/dataswamp/swampfile"
-	"path"
+	"github.com/matryer/is"
 	"os"
+	"path"
 	"testing"
-	"github.com/matryer/is"
 )
 
 func newRepoDir(t *testing.T) string {
@@ -19,7 +19,6 @@ func newRepoDir(t *testing.T) string {
 	return d
 }
 
-
 func TestFsFileRepo(t *testing.T) {
 
 	var fac = func() swampfile.Repository {
@@ -34,7 +33,6 @@ func TestFsFileRepo(t *testing.T) {
 	swampfile.RepositoryContract(fac, t)
 }
 
-
 func TestEmptyDir(t *testing.T) {
 	is := is.New(t)
 	dir_path := newRepoDir(t)
@@ -46,7 +44,6 @@ func TestEmptyDir(t *testing.T) {
 	is.NoErr(err)
 }
 
-
 func TestDirtyDir(t *testing.T) {
 	is := is.New(t)
 	dir_path := newRepoDir(t)
@@ -59,7 +56,6 @@ func TestDirtyDir(t *testing.T) {
 	is.Equal(err, ErrDirtyRepo)
 }
 
-
 func TestDirtyWithFlag(t *testing.T) {
 	is := is.New(t)
 	dir_path := newRepoDir(t)

infrastructure/memory/swampfile/repository.go

@@ -1,16 +1,14 @@
 package swampfile
 
 import (
+	"caj-larsson/bog/dataswamp/swampfile"
+	"github.com/spf13/afero"
+	"io/fs"
 	"os"
 	"path"
 	"time"
-	"io/fs"
-	"caj-larsson/bog/dataswamp/swampfile"
-	"github.com/spf13/afero"
 )
 
-
-
 type SwampFile struct {
 	filename string
 	file     afero.File
@@ -48,7 +46,7 @@ func (f SwampFile) Modified() time.Time {
 
 // The actual repository
 type Repository struct {
-	fs afero.Fs
+	Fs afero.Fs
 }
 
 func NewRepository() swampfile.Repository {
@@ -58,8 +56,8 @@ func NewRepository() swampfile.Repository {
 func (r Repository) Create(filename string, namespace_stub string) (swampfile.SwampInFile, error) {
 	abs_path := path.Join(namespace_stub, filename)
 	dir := path.Dir(abs_path)
-	r.fs.MkdirAll(dir, 0750)
+	r.Fs.MkdirAll(dir, 0750)
-	file, err := r.fs.OpenFile(abs_path, os.O_RDWR|os.O_CREATE, 0644)
+	file, err := r.Fs.OpenFile(abs_path, os.O_RDWR|os.O_CREATE, 0644)
 
 	if err != nil {
 		panic(err)
@@ -74,8 +72,8 @@ func (r Repository) Open(filename string, namespace_stub string) (swampfile.Swam
 	abs_path := path.Join(namespace_stub, filename)
 
 	dir := path.Dir(abs_path)
-	r.fs.MkdirAll(dir, 0750)
+	r.Fs.MkdirAll(dir, 0750)
-	file, err := r.fs.OpenFile(abs_path, os.O_RDONLY, 0644)
+	file, err := r.Fs.OpenFile(abs_path, os.O_RDONLY, 0644)
 
 	if err != nil {
 		return nil, swampfile.ErrNotExists
@@ -87,26 +85,29 @@ func (r Repository) Open(filename string, namespace_stub string) (swampfile.Swam
 
 func (r Repository) Delete(filename string, namespace_stub string) {
 	abs_path := path.Join(namespace_stub, filename)
-	r.fs.Remove(abs_path)
+	r.Fs.Remove(abs_path)
 }
 
-func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) error {
+func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) ([]swampfile.DeletedSwampFile, error) {
-	err := afero.Walk(r.fs, namespace_stub, func(path string, info fs.FileInfo, err error) error {
+	df := []swampfile.DeletedSwampFile{}
+
+	err := afero.Walk(r.Fs, namespace_stub, func(path string, info fs.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
-		if info.Mode().IsRegular() && ts.Before(info.ModTime()) {
+		if info.Mode().IsRegular() {
-			r.fs.Remove(path)
+			if info.ModTime().Before(ts) {
+				df = append(df, swampfile.DeletedSwampFile{path, info.Size()})
+				r.Fs.Remove(path)
+			}
 			return nil
 		}
 
 		if !info.Mode().IsDir() {
 			return swampfile.ErrCorrupted
 		}
-
 		return nil
 	})
-
+	return df, err
-	return err
 }