swampfile.Repository.DeleteOlderThan

master
Caj Larsson 3 years ago
parent 7bb9bf90c2
commit 2ce12745a6

@ -4,11 +4,10 @@ import (
"caj-larsson/bog/dataswamp/namespace"
"caj-larsson/bog/dataswamp/swampfile"
"io"
"path"
"path/filepath"
"strconv"
"strings"
"time"
// "errors"
// "fmt"
)
type SwampFileService struct {
@ -82,7 +81,7 @@ func (s SwampFileService) SaveFile(ref swampfile.FileReference, src io.Reader, s
var buf = make([]byte, 1)
overread, err :=src.Read(buf)
overread, err := src.Read(buf)
if overread > 0 || err != io.EOF {
s.swamp_file_repo.Delete(ref.Path, strconv.FormatInt(ns.ID, 10))
@ -114,6 +113,28 @@ func (s SwampFileService) OpenOutFile(ref swampfile.FileReference) (swampfile.Sw
return f, nil
}
func CleanPath(inpath string) string {
return filepath.FromSlash(path.Clean("/" + strings.Trim(inpath, "/")))
func (s SwampFileService) CleanUpExpiredFiles() error {
nss, err := s.namespace_repo.All()
if err != nil {
return err
}
for _, ns := range nss {
expiry := time.Now().Add(-ns.AllowanceDuration)
dfs, err := s.swamp_file_repo.DeleteOlderThan(strconv.FormatInt(ns.ID, 10), expiry)
// panic(errors.New(fmt.Sprintf("%+v", dfs)))
for _, df := range dfs {
ns.FileQuota.Remove(df.Size)
}
if err != nil {
panic(err)
}
s.namespace_repo.Update(ns.ID, ns)
}
return nil
}

@ -4,6 +4,7 @@ import (
"bytes"
"caj-larsson/bog/dataswamp/swampfile"
"github.com/matryer/is"
"github.com/spf13/afero"
"testing"
"time"
// "caj-larsson/bog/dataswamp/namespace"
@ -11,7 +12,7 @@ import (
m_swampfile "caj-larsson/bog/infrastructure/memory/swampfile"
)
var file_ref1 = swampfile.FileReference{"/ptah1", "ns1"}
var file_ref1 = swampfile.FileReference{"/path1", "ns1"}
var file_ref2 = swampfile.FileReference{"/path1", "ns2"}
var file_ref3 = swampfile.FileReference{"/path2", "ns1"}
@ -85,14 +86,13 @@ func TestNSIsolation(t *testing.T) {
is.Equal(outfile.String(), "My bog data ns1")
}
func TestPathStrictMode(t *testing.T) {
is := is.New(t)
s := NewTestSwampFileService()
ns_file := bytes.NewBufferString("My bog data ns1")
ref := swampfile.FileReference {
ref := swampfile.FileReference{
"/path/../with/../backrefs",
"ns1",
}
@ -106,7 +106,6 @@ func TestPathStrictMode(t *testing.T) {
is.Equal(err, swampfile.ErrUnacceptablePath)
}
func TestQuotaWithContenSizeLieOver(t *testing.T) {
is := is.New(t)
s := NewTestSwampFileService()
@ -122,7 +121,6 @@ func TestQuotaWithContenSizeLieOver(t *testing.T) {
is.Equal(err, swampfile.ErrContentSizeExceeded)
}
func TestQuotaWithContenSizeLieUnder(t *testing.T) {
is := is.New(t)
s := NewTestSwampFileService()
@ -133,3 +131,29 @@ func TestQuotaWithContenSizeLieUnder(t *testing.T) {
is.Equal(err, swampfile.ErrContentSizeExaggerated)
}
func TestCleanUpExpired(t *testing.T) {
is := is.New(t)
fs := afero.NewMemMapFs()
file_repo := m_swampfile.Repository{fs}
ns_repo := m_namespace.NewRepository()
s := NewSwampFileService(ns_repo, file_repo, 1024, time.Hour)
fakefile := bytes.NewBufferString("My bog data")
err := s.SaveFile(file_ref1, fakefile, int64(fakefile.Len()))
is.NoErr(err)
fakefile = bytes.NewBufferString("My bog data")
err = s.SaveFile(file_ref3, fakefile, int64(fakefile.Len()))
is.NoErr(err)
err = fs.Chtimes("1/path1", time.Now(), time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))
is.NoErr(err)
is.NoErr(s.CleanUpExpiredFiles())
ns, err := ns_repo.GetByName("ns1")
is.NoErr(err)
is.Equal(ns.FileQuota.CurrentUsage, int64(len("My bog data")))
}

@ -57,7 +57,7 @@ func basicFileOperationContract(fac func() Repository, t *testing.T) {
expiring_file.Write([]byte(testdata))
expiring_file.Close()
err = repo.DeleteOlderThan("ua1", time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))
_, err = repo.DeleteOlderThan("ua1", time.Now().Add(time.Hour))
is.NoErr(err)
expired_file, err := repo.Open("deep/dir/expiring.new", "ua1")

@ -6,5 +6,5 @@ type Repository interface {
Create(filename string, namespace_stub string) (SwampInFile, error)
Open(filename string, namespace_stub string) (SwampOutFile, error)
Delete(filename string, namespace_stub string)
DeleteOlderThan(namespace_stub string, ts time.Time) error
DeleteOlderThan(namespace_stub string, ts time.Time) ([]DeletedSwampFile, error)
}

@ -31,6 +31,11 @@ type FileReference struct {
UserAgent string
}
type DeletedSwampFile struct {
Path string
Size int64
}
func (fr *FileReference) Clean(strict bool) error {
c := filepath.FromSlash(path.Clean("/" + strings.Trim(fr.Path, "/")))

@ -2,12 +2,12 @@ package swampfile
import (
"caj-larsson/bog/dataswamp/swampfile"
"errors"
"io/fs"
"os"
"path"
"path/filepath"
"io/fs"
"time"
"errors"
)
var ErrDirtyRepo = errors.New("Dirty repository without flag")
@ -44,7 +44,8 @@ func (f FileSystemSwampFileData) Size() int64 {
}
func (f FileSystemSwampFileData) Modified() time.Time {
return time.Now()
stat, _ := f.file.Stat()
return stat.ModTime()
}
type Repository struct {
@ -75,10 +76,9 @@ func NewRepository(root string) (*Repository, error) {
return nil, ErrDirtyRepo
}
flagfile.Close()
return &Repository { root }, nil
return &Repository{root}, nil
}
func (f Repository) absPath(filename string, namespace_ns string) string {
return path.Join(f.Root, namespace_ns, filename)
}
@ -125,7 +125,9 @@ func (f Repository) Delete(filename string, namespace_ns string) {
os.Remove(abs_path)
}
func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) error {
func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) ([]swampfile.DeletedSwampFile, error) {
df := []swampfile.DeletedSwampFile{}
dr := path.Join(r.Root, namespace_stub)
err := filepath.Walk(dr, func(path string, info fs.FileInfo, err error) error {
if err != nil {
@ -134,8 +136,8 @@ func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) error {
mode := info.Mode()
if mode.IsRegular() {
if ts.Before(info.ModTime()) {
if info.ModTime().Before(ts) {
df = append(df, swampfile.DeletedSwampFile{path, info.Size()})
err = os.Remove(path)
if err != nil {
panic(err)
@ -150,5 +152,5 @@ func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) error {
return nil
})
return err
return df, err
}

@ -2,10 +2,10 @@ package swampfile
import (
"caj-larsson/bog/dataswamp/swampfile"
"path"
"github.com/matryer/is"
"os"
"path"
"testing"
"github.com/matryer/is"
)
func newRepoDir(t *testing.T) string {
@ -19,7 +19,6 @@ func newRepoDir(t *testing.T) string {
return d
}
func TestFsFileRepo(t *testing.T) {
var fac = func() swampfile.Repository {
@ -34,7 +33,6 @@ func TestFsFileRepo(t *testing.T) {
swampfile.RepositoryContract(fac, t)
}
func TestEmptyDir(t *testing.T) {
is := is.New(t)
dir_path := newRepoDir(t)
@ -46,7 +44,6 @@ func TestEmptyDir(t *testing.T) {
is.NoErr(err)
}
func TestDirtyDir(t *testing.T) {
is := is.New(t)
dir_path := newRepoDir(t)
@ -59,7 +56,6 @@ func TestDirtyDir(t *testing.T) {
is.Equal(err, ErrDirtyRepo)
}
func TestDirtyWithFlag(t *testing.T) {
is := is.New(t)
dir_path := newRepoDir(t)

@ -1,16 +1,14 @@
package swampfile
import (
"caj-larsson/bog/dataswamp/swampfile"
"github.com/spf13/afero"
"io/fs"
"os"
"path"
"time"
"io/fs"
"caj-larsson/bog/dataswamp/swampfile"
"github.com/spf13/afero"
)
type SwampFile struct {
filename string
file afero.File
@ -48,7 +46,7 @@ func (f SwampFile) Modified() time.Time {
// The actual repository
type Repository struct {
fs afero.Fs
Fs afero.Fs
}
func NewRepository() swampfile.Repository {
@ -58,8 +56,8 @@ func NewRepository() swampfile.Repository {
func (r Repository) Create(filename string, namespace_stub string) (swampfile.SwampInFile, error) {
abs_path := path.Join(namespace_stub, filename)
dir := path.Dir(abs_path)
r.fs.MkdirAll(dir, 0750)
file, err := r.fs.OpenFile(abs_path, os.O_RDWR|os.O_CREATE, 0644)
r.Fs.MkdirAll(dir, 0750)
file, err := r.Fs.OpenFile(abs_path, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
panic(err)
@ -74,8 +72,8 @@ func (r Repository) Open(filename string, namespace_stub string) (swampfile.Swam
abs_path := path.Join(namespace_stub, filename)
dir := path.Dir(abs_path)
r.fs.MkdirAll(dir, 0750)
file, err := r.fs.OpenFile(abs_path, os.O_RDONLY, 0644)
r.Fs.MkdirAll(dir, 0750)
file, err := r.Fs.OpenFile(abs_path, os.O_RDONLY, 0644)
if err != nil {
return nil, swampfile.ErrNotExists
@ -87,26 +85,29 @@ func (r Repository) Open(filename string, namespace_stub string) (swampfile.Swam
func (r Repository) Delete(filename string, namespace_stub string) {
abs_path := path.Join(namespace_stub, filename)
r.fs.Remove(abs_path)
r.Fs.Remove(abs_path)
}
func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) error {
err := afero.Walk(r.fs, namespace_stub, func(path string, info fs.FileInfo, err error) error {
func (r Repository) DeleteOlderThan(namespace_stub string, ts time.Time) ([]swampfile.DeletedSwampFile, error) {
df := []swampfile.DeletedSwampFile{}
err := afero.Walk(r.Fs, namespace_stub, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode().IsRegular() && ts.Before(info.ModTime()) {
r.fs.Remove(path)
if info.Mode().IsRegular() {
if info.ModTime().Before(ts) {
df = append(df, swampfile.DeletedSwampFile{path, info.Size()})
r.Fs.Remove(path)
}
return nil
}
if !info.Mode().IsDir() {
return swampfile.ErrCorrupted
}
return nil
})
return err
return df, err
}

Loading…
Cancel
Save