Compare commits

..

No commits in common. 'master' and 'v0.2.0' have entirely different histories.

@ -1,2 +0,0 @@
Dockerfile
test

1
.gitignore vendored

@ -1,3 +1,2 @@
sql.db sql.db
bog bog
test

@ -1,7 +0,0 @@
pipeline:
build:
image: golang
commands:
- go test ./...
environment:
- GOPRIVATE=git.sg.caj.me/caj

@ -1,41 +0,0 @@
FROM golang:alpine as builder
ENV GO111MODULE=on
ENV USER=appuser
ENV UID=1000
RUN adduser \
--disabled-password \
--gecos "" \
--home "/nonexistent" \
--shell "/sbin/nologin" \
--no-create-home \
--uid "${UID}" \
"${USER}"
RUN apk update && apk add --no-cache git ca-certificates tzdata sqlite build-base
RUN mkdir /build
COPY . /build/
WORKDIR /build
COPY ./ ./
RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o bog .
FROM scratch AS final
LABEL author="Cajually <me@caj.me>"
COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo
COPY --from=builder /etc/passwd /etc/passwd
COPY --from=builder /etc/group /etc/group
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY --from=builder /build/bog /
COPY --from=builder /build/server/views /server/views
COPY --from=builder /build/default.toml /
WORKDIR /
USER appuser:appuser
ENTRYPOINT ["/bog"]
EXPOSE 8002

@ -1,7 +1,4 @@
[![status-badge](https://ci.sg.caj.me/api/badges/caj/bog/status.svg)](https://ci.sg.caj.me/caj/bog)
# Bog: The Dataswamp # Bog: The Dataswamp
![Logo](https://git.sg.caj.me/caj/bog/raw/branch/master/doc/logo.jpg)
Is the warehouse too strict? Maybe you don't know the schema yet. And Is the warehouse too strict? Maybe you don't know the schema yet. And
you know your lake is already full with a bunch of stuff that should you know your lake is already full with a bunch of stuff that should
have been deleted long ago? have been deleted long ago?

@ -1,7 +0,0 @@
package dataswamp
import (
// "caj-larsson/bog/dataswamp/namespace"
)
type AdminService struct{}

@ -1 +0,0 @@
package dataswamp

@ -11,8 +11,6 @@ type Namespace struct {
LastSeen time.Time LastSeen time.Time
AllowanceDuration time.Duration AllowanceDuration time.Duration
FileQuota FileSizeQuota FileQuota FileSizeQuota
Download FileStat
Upload FileStat
} }
var ( var (

@ -1,57 +0,0 @@
package namespace
import (
"github.com/matryer/is"
"testing"
"time"
)
func RepositoryContract(fac func() Repository, t *testing.T) {
basicNamespaceContract(fac, t)
}
func basicNamespaceContract(fac func() Repository, t *testing.T) {
is := is.New(t)
r := fac()
all, err := r.All()
is.NoErr(err)
is.Equal(len(all), 0)
ns := Namespace{
23,
"n1",
time.Now(),
time.Duration(time.Hour * 3),
FileSizeQuota{1000, 0},
FileStat{1, 2},
FileStat{3, 4},
}
ns1, err := r.Create(ns)
is.NoErr(err)
ns.Name = "n2"
ns2, err := r.Create(ns)
is.NoErr(err)
is.True(ns1.ID != ns2.ID)
all, err = r.All()
is.NoErr(err)
is.Equal(len(all), 2)
is.Equal(ns.ID, int64(23))
ns3, _ := r.GetByName("n2")
is.Equal(*ns3, *ns2)
is.NoErr(r.Delete(ns2.ID))
all, err = r.All()
is.NoErr(err)
is.Equal(len(all), 1)
}

@ -1,111 +0,0 @@
package namespace
import (
"caj-larsson/bog/util"
"fmt"
"time"
)
type Clock interface {
Now() time.Time
}
type NamespaceService struct {
repo Repository
outboxes []func(util.Event)
logger util.Logger
clock Clock
default_ttl time.Duration
default_quota_bytes int64
}
func NewNamespaceService(repo Repository, logger util.Logger, clock Clock, default_ttl time.Duration, default_quota_bytes int64) *NamespaceService {
return &NamespaceService{
repo,
nil,
logger,
clock,
default_ttl,
default_quota_bytes,
}
}
func (s *NamespaceService) GetOrCreateNs(name string) *Namespace {
ns, err := s.repo.GetByName(name)
if err == ErrNotExists {
new_ns := Namespace{
0,
name,
s.clock.Now(),
s.default_ttl,
FileSizeQuota{s.default_quota_bytes, 0},
FileStat{0, 0},
FileStat{0, 0},
}
created_ns, err := s.repo.Create(new_ns)
if err != nil {
panic(err)
}
return created_ns
}
if err != nil {
panic(err)
}
return ns
}
func (s *NamespaceService) Wire(reg func(string, util.EventHandler), outbox func(ev util.Event)) {
reg("FileUsed", s.handleFileUsed)
s.outboxes = append(s.outboxes, outbox)
reg("FileUsed", s.handleFileUsed)
reg("FileDeleted", s.handleFileDeleted)
reg("FileRecieved", s.handleFileRecieved)
}
func (s *NamespaceService) All() []Namespace {
nss, err := s.repo.All()
if err != nil {
panic(err)
}
return nss
}
func (s *NamespaceService) handleFileUsed(payload interface{}) {
var payload_s = payload.(struct {
Name string
Size int64
})
fmt.Printf("file used %v\n", payload_s)
ns := s.GetOrCreateNs(payload_s.Name)
ns.FileQuota = ns.FileQuota.Add(payload_s.Size)
s.repo.Update(ns.ID, *ns)
}
func (s *NamespaceService) handleFileDeleted(payload interface{}) {
var payload_s = payload.(struct {
Name string
Size int64
})
fmt.Printf("file deleted %v\n", payload_s)
ns := s.GetOrCreateNs(payload_s.Name)
ns.FileQuota = ns.FileQuota.Add(-payload_s.Size)
fmt.Printf("file usage %v\n", ns.FileQuota)
s.repo.Update(ns.ID, *ns)
}
func (s *NamespaceService) handleFileRecieved(payload interface{}) {
var payload_s = payload.(struct {
Name string
Size int64
})
fmt.Printf("file recieved %v\n", payload_s)
ns := s.GetOrCreateNs(payload_s.Name)
ns.FileQuota = ns.FileQuota.Add(payload_s.Size)
fmt.Printf("file usage %v\n", ns.FileQuota)
s.repo.Update(ns.ID, *ns)
}

@ -1,38 +0,0 @@
package namespace
import (
"caj-larsson/bog/util"
"testing"
)
func TestEventTest(t *testing.T) {
eb := util.NewEventBus()
svc := NamespaceService{}
svc.Wire(eb.Register, eb.Handle)
events := []util.Event{
*util.NewEvent("FileUsed", struct {
Name string
Size int64
}{
"asd",
int64(12),
}),
*util.NewEvent("FileDeleted", struct {
Name string
Size int64
}{
"asd",
int64(12),
}),
*util.NewEvent("FileRecieved", struct {
Name string
Size int64
}{
"asd",
int64(12),
}),
}
util.AcceptsMessage(t, eb, events)
}

@ -5,36 +5,30 @@ type FileSizeQuota struct {
CurrentUsage int64 CurrentUsage int64
} }
func (f FileSizeQuota) Allows(size int64) bool { func (f *FileSizeQuota) Allows(size int64) bool {
return f.Remaining() >= size return f.Remaining() >= size
} }
func (f FileSizeQuota) Remaining() int64 { func (f *FileSizeQuota) Remaining() int64 {
return f.AllowanceKB - f.CurrentUsage return f.AllowanceKB - f.CurrentUsage
} }
func (f FileSizeQuota) Add(size int64) FileSizeQuota { func (f *FileSizeQuota) Add(size int64) error {
return FileSizeQuota{ if !f.Allows(size) {
f.AllowanceKB, return ErrExceedQuota
f.CurrentUsage + size,
}
} }
func (f FileSizeQuota) Remove(size int64) FileSizeQuota { f.CurrentUsage += size
return FileSizeQuota{
f.AllowanceKB,
f.CurrentUsage - size,
}
}
type FileStat struct { return nil
Num int64
SizeB int64
} }
func (s FileStat) Add(size int64) FileStat { func (f *FileSizeQuota) Remove(size int64) error {
return FileStat{ if size > f.CurrentUsage {
s.Num + 1, return ErrQuotaInvalid
s.SizeB + size,
} }
f.CurrentUsage -= size
return nil
} }

@ -16,8 +16,22 @@ func TestQuota(t *testing.T) {
func TestQuotaManipulation(t *testing.T) { func TestQuotaManipulation(t *testing.T) {
is := is.New(t) is := is.New(t)
quota := FileSizeQuota{1000, 0} quota := FileSizeQuota{1000, 0}
quota = quota.Add(500)
is.NoErr(quota.Add(500))
is.Equal(quota.CurrentUsage, int64(500)) is.Equal(quota.CurrentUsage, int64(500))
quota = quota.Remove(1000)
is.Equal(quota.CurrentUsage, int64(-500)) is.NoErr(quota.Add(500))
is.Equal(quota.Add(1), ErrExceedQuota)
is.Equal(quota.CurrentUsage, int64(1000))
is.Equal(quota.Remove(1001), ErrQuotaInvalid)
is.Equal(quota.CurrentUsage, int64(1000))
is.NoErr(quota.Remove(1000))
is.Equal(quota.CurrentUsage, int64(0))
} }

@ -0,0 +1,138 @@
package dataswamp
import (
"caj-larsson/bog/dataswamp/namespace"
"caj-larsson/bog/dataswamp/swampfile"
"io"
"strconv"
"time"
// "errors"
// "fmt"
)
type SwampFileService struct {
namespace_repo namespace.Repository
swamp_file_repo swampfile.Repository
default_allowance_bytes int64
default_allowance_duration time.Duration
}
func NewSwampFileService(
namespace_repo namespace.Repository,
swamp_file_repo swampfile.Repository,
da_bytes int64,
da_duration time.Duration,
) SwampFileService {
return SwampFileService{namespace_repo, swamp_file_repo, da_bytes, da_duration}
}
func (s SwampFileService) getOrCreateNs(namespace_in string) *namespace.Namespace {
ns, err := s.namespace_repo.GetByName(namespace_in)
if err == namespace.ErrNotExists {
new_ns := namespace.Namespace{
0,
namespace_in,
time.Now(),
s.default_allowance_duration,
namespace.FileSizeQuota{s.default_allowance_bytes, 0},
}
created_ns, err := s.namespace_repo.Create(new_ns)
if err != nil {
panic(err)
}
return created_ns
}
if err != nil {
panic(err)
}
return ns
}
func (s SwampFileService) SaveFile(ref swampfile.FileReference, src io.Reader, size int64) error {
ns := s.getOrCreateNs(ref.UserAgent)
err := ref.Clean(true)
if err != nil {
return err
}
if !ns.FileQuota.Allows(size) {
return namespace.ErrExceedQuota
}
f, err := s.swamp_file_repo.Create(ref.Path, strconv.FormatInt(ns.ID, 10))
if err != nil {
return err
}
written, err := io.CopyN(f, src, size)
if written < size {
s.swamp_file_repo.Delete(ref.Path, strconv.FormatInt(ns.ID, 10))
return swampfile.ErrContentSizeExaggerated
}
var buf = make([]byte, 1)
overread, err := src.Read(buf)
if overread > 0 || err != io.EOF {
s.swamp_file_repo.Delete(ref.Path, strconv.FormatInt(ns.ID, 10))
return swampfile.ErrContentSizeExceeded
}
f.Close()
ns.FileQuota.Add(size)
s.namespace_repo.Update(ns.ID, *ns)
return nil
}
func (s SwampFileService) OpenOutFile(ref swampfile.FileReference) (swampfile.SwampOutFile, error) {
ns := s.getOrCreateNs(ref.UserAgent)
err := ref.Clean(true)
if err != nil {
return nil, err
}
f, err := s.swamp_file_repo.Open(ref.Path, strconv.FormatInt(ns.ID, 10))
if err != nil {
return nil, err
}
return f, nil
}
func (s SwampFileService) CleanUpExpiredFiles() error {
nss, err := s.namespace_repo.All()
if err != nil {
return err
}
for _, ns := range nss {
expiry := time.Now().Add(-ns.AllowanceDuration)
dfs, err := s.swamp_file_repo.DeleteOlderThan(strconv.FormatInt(ns.ID, 10), expiry)
for _, df := range dfs {
ns.FileQuota.Remove(df.Size)
}
if err != nil {
panic(err)
}
s.namespace_repo.Update(ns.ID, ns)
}
return nil
}

@ -2,47 +2,29 @@ package dataswamp
import ( import (
"bytes" "bytes"
"caj-larsson/bog/dataswamp/namespace"
"caj-larsson/bog/dataswamp/swampfile" "caj-larsson/bog/dataswamp/swampfile"
m_namespace "caj-larsson/bog/infrastructure/memory/namespace"
m_swampfile "caj-larsson/bog/infrastructure/memory/swampfile"
"caj-larsson/bog/infrastructure/system_time"
"fmt"
"github.com/matryer/is" "github.com/matryer/is"
"github.com/spf13/afero" "github.com/spf13/afero"
"testing" "testing"
"time" "time"
// "caj-larsson/bog/dataswamp/namespace"
m_namespace "caj-larsson/bog/infrastructure/memory/namespace"
m_swampfile "caj-larsson/bog/infrastructure/memory/swampfile"
) )
type TestLogger struct{}
func (t TestLogger) Debug(format string, a ...interface{}) {}
func (t TestLogger) Info(format string, a ...interface{}) {}
func (t TestLogger) Warn(format string, a ...interface{}) {}
var file_ref1 = swampfile.FileReference{"/path1", "ns1"} var file_ref1 = swampfile.FileReference{"/path1", "ns1"}
var file_ref2 = swampfile.FileReference{"/path1", "ns2"} var file_ref2 = swampfile.FileReference{"/path1", "ns2"}
var file_ref3 = swampfile.FileReference{"/path2", "ns1"} var file_ref3 = swampfile.FileReference{"/path2", "ns1"}
func NewTestDataSwampService() DataSwampService { func NewTestSwampFileService() SwampFileService {
file_repo := m_swampfile.NewRepository() file_repo := m_swampfile.NewRepository()
ns_repo := m_namespace.NewRepository() ns_repo := m_namespace.NewRepository()
return NewSwampFileService(ns_repo, file_repo, 1024, time.Hour)
logger := TestLogger{}
ns_svc := namespace.NewNamespaceService(
ns_repo,
logger,
system_time.Clock{},
time.Hour,
1024,
)
return *NewDataSwampService(*ns_svc, file_repo, logger)
} }
func TestFileDontExist(t *testing.T) { func TestFileDontExist(t *testing.T) {
is := is.New(t) is := is.New(t)
s := NewTestDataSwampService() s := NewTestSwampFileService()
outfile, err := s.OpenOutFile(file_ref1) outfile, err := s.OpenOutFile(file_ref1)
is.True(err == swampfile.ErrNotExists) is.True(err == swampfile.ErrNotExists)
@ -51,7 +33,7 @@ func TestFileDontExist(t *testing.T) {
func TestFileIsStored(t *testing.T) { func TestFileIsStored(t *testing.T) {
is := is.New(t) is := is.New(t)
s := NewTestDataSwampService() s := NewTestSwampFileService()
fakefile := bytes.NewBufferString("My bog data") fakefile := bytes.NewBufferString("My bog data")
@ -72,7 +54,7 @@ func TestFileIsStored(t *testing.T) {
func TestFileIsReadBack(t *testing.T) { func TestFileIsReadBack(t *testing.T) {
is := is.New(t) is := is.New(t)
s := NewTestDataSwampService() s := NewTestSwampFileService()
infile := bytes.NewBufferString("My bog data") infile := bytes.NewBufferString("My bog data")
@ -88,7 +70,7 @@ func TestFileIsReadBack(t *testing.T) {
func TestNSIsolation(t *testing.T) { func TestNSIsolation(t *testing.T) {
is := is.New(t) is := is.New(t)
s := NewTestDataSwampService() s := NewTestSwampFileService()
ns1_file := bytes.NewBufferString("My bog data ns1") ns1_file := bytes.NewBufferString("My bog data ns1")
ns2_file := bytes.NewBufferString("My bog data ns2") ns2_file := bytes.NewBufferString("My bog data ns2")
@ -106,7 +88,7 @@ func TestNSIsolation(t *testing.T) {
func TestPathStrictMode(t *testing.T) { func TestPathStrictMode(t *testing.T) {
is := is.New(t) is := is.New(t)
s := NewTestDataSwampService() s := NewTestSwampFileService()
ns_file := bytes.NewBufferString("My bog data ns1") ns_file := bytes.NewBufferString("My bog data ns1")
@ -126,7 +108,7 @@ func TestPathStrictMode(t *testing.T) {
func TestQuotaWithContenSizeLieOver(t *testing.T) { func TestQuotaWithContenSizeLieOver(t *testing.T) {
is := is.New(t) is := is.New(t)
s := NewTestDataSwampService() s := NewTestSwampFileService()
largefakefile := bytes.NewBufferString("") largefakefile := bytes.NewBufferString("")
@ -141,7 +123,7 @@ func TestQuotaWithContenSizeLieOver(t *testing.T) {
func TestQuotaWithContenSizeLieUnder(t *testing.T) { func TestQuotaWithContenSizeLieUnder(t *testing.T) {
is := is.New(t) is := is.New(t)
s := NewTestDataSwampService() s := NewTestSwampFileService()
largefakefile := bytes.NewBufferString("small") largefakefile := bytes.NewBufferString("small")
@ -156,32 +138,22 @@ func TestCleanUpExpired(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
file_repo := m_swampfile.Repository{fs} file_repo := m_swampfile.Repository{fs}
ns_repo := m_namespace.NewRepository() ns_repo := m_namespace.NewRepository()
logger := TestLogger{} s := NewSwampFileService(ns_repo, file_repo, 1024, time.Hour)
ns_svc := namespace.NewNamespaceService(
ns_repo,
logger,
system_time.Clock{},
time.Hour,
1024,
)
s := NewDataSwampService(*ns_svc, file_repo, logger) fakefile := bytes.NewBufferString("My bog data")
fakefile := bytes.NewBufferString("My bog")
err := s.SaveFile(file_ref1, fakefile, int64(fakefile.Len())) err := s.SaveFile(file_ref1, fakefile, int64(fakefile.Len()))
is.NoErr(err) is.NoErr(err)
fakefile = bytes.NewBufferString("My bog") fakefile = bytes.NewBufferString("My bog data")
err = s.SaveFile(file_ref3, fakefile, int64(fakefile.Len())) err = s.SaveFile(file_ref3, fakefile, int64(fakefile.Len()))
is.NoErr(err) is.NoErr(err)
err = fs.Chtimes("1/path1", time.Now(), time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)) err = fs.Chtimes("1/path1", time.Now(), time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))
is.NoErr(err)
is.NoErr(s.CleanUpExpiredFiles()) is.NoErr(s.CleanUpExpiredFiles())
ns, err := ns_repo.GetByName("ns1") ns, err := ns_repo.GetByName("ns1")
fmt.Printf("file final usage %v\n", ns.FileQuota)
is.NoErr(err) is.NoErr(err)
fmt.Printf("file\n")
is.Equal(ns.FileQuota.CurrentUsage, int64(len("My bog"))) is.Equal(ns.FileQuota.CurrentUsage, int64(len("My bog data")))
} }

@ -1,138 +0,0 @@
package dataswamp
import (
"caj-larsson/bog/dataswamp/namespace"
"caj-larsson/bog/dataswamp/swampfile"
"caj-larsson/bog/util"
"io"
"strconv"
"time"
// "errors"
// "fmt"
)
type DataSwampService struct {
ns_svc namespace.NamespaceService
swamp_file_repo swampfile.Repository
logger util.Logger
eventBus util.EventBus
}
func NewDataSwampService(
ns_svc namespace.NamespaceService,
swamp_file_repo swampfile.Repository,
logger util.Logger,
) *DataSwampService {
s := DataSwampService{ns_svc, swamp_file_repo, logger, *util.NewEventBus()}
ns_svc.Wire(s.eventBus.Register, s.eventBus.Handle)
return &s
}
func (s DataSwampService) NamespaceStats() []namespace.Namespace {
return s.ns_svc.All()
}
func (s DataSwampService) SaveFile(ref swampfile.FileReference, src io.Reader, size int64) error {
ns := s.ns_svc.GetOrCreateNs(ref.UserAgent)
r, err := ref.Clean(true)
if err != nil {
return err
}
if !ns.FileQuota.Allows(size) {
return namespace.ErrExceedQuota
}
f, err := s.swamp_file_repo.Create(r.Path, strconv.FormatInt(ns.ID, 10))
if err != nil {
// TODO: convert this into a different error.
return err
}
s.eventBus.Handle(*util.NewEvent("FileUsed", struct {
Name string
Size int64
}{
ns.Name,
f.Size(),
}))
// TODO: rewrite this into an interruptable loop that emits downloaded events
written, err := io.CopyN(f, src, size)
if written < size {
s.swamp_file_repo.Delete(r.Path, strconv.FormatInt(ns.ID, 10)) //
return swampfile.ErrContentSizeExaggerated
}
var buf = make([]byte, 1)
overread, err := src.Read(buf)
if overread > 0 || err != io.EOF {
s.swamp_file_repo.Delete(r.Path, strconv.FormatInt(ns.ID, 10))
return swampfile.ErrContentSizeExceeded
}
err = f.Close()
if err != nil {
panic(err)
}
s.eventBus.Handle(*util.NewEvent("FileRecieved", struct {
Name string
Size int64
}{
ns.Name,
written,
}))
return nil
}
func (s DataSwampService) OpenOutFile(ref swampfile.FileReference) (swampfile.SwampOutFile, error) {
ns := s.ns_svc.GetOrCreateNs(ref.UserAgent)
r, err := ref.Clean(true)
if err != nil {
return nil, err
}
f, err := s.swamp_file_repo.Open(r.Path, strconv.FormatInt(ns.ID, 10))
if err != nil {
return nil, err
}
s.eventBus.Handle(*util.NewEvent("FileSent", f.Size()))
return f, nil
}
func (s DataSwampService) CleanUpExpiredFiles() error {
s.logger.Info("Cleaning up expired files")
for _, ns := range s.ns_svc.All() {
expiry := time.Now().Add(-ns.AllowanceDuration)
dfs, err := s.swamp_file_repo.DeleteOlderThan(strconv.FormatInt(ns.ID, 10), expiry)
if err != nil {
panic(err)
}
for _, df := range dfs {
s.eventBus.Handle(*util.NewEvent("FileDeleted", struct {
Name string
Size int64
}{
ns.Name,
df.Size,
}))
}
}
return nil
}

@ -33,7 +33,6 @@ func basicFileOperationContract(fac func() Repository, t *testing.T) {
is.NoErr(err) is.NoErr(err)
is.True(reopened_file != nil) is.True(reopened_file != nil)
is.Equal(reopened_file.Size(), int64(len(testdata)))
readback := make([]byte, 128) readback := make([]byte, 128)

@ -36,17 +36,14 @@ type DeletedSwampFile struct {
Size int64 Size int64
} }
func (fr *FileReference) Clean(strict bool) (*FileReference, error) { func (fr *FileReference) Clean(strict bool) error {
c := filepath.FromSlash(path.Clean("/" + strings.Trim(fr.Path, "/"))) c := filepath.FromSlash(path.Clean("/" + strings.Trim(fr.Path, "/")))
if c != fr.Path && strict { if c != fr.Path && strict {
return nil, ErrUnacceptablePath return ErrUnacceptablePath
} }
n := FileReference{ fr.Path = c
c,
fr.UserAgent,
}
return &n, nil return nil
} }

@ -25,9 +25,9 @@ func TestSwampPathNotStrict(t *testing.T) {
"ns", "ns",
} }
rc, err := rf.Clean(false) err := rf.Clean(false)
is.NoErr(err) is.NoErr(err)
is.Equal(rc.Path, tc.clean) is.Equal(rf.Path, tc.clean)
} }
} }
@ -39,6 +39,6 @@ func TestSwampPathStrict(t *testing.T) {
"ns", "ns",
} }
_, err := rf.Clean(true) err := rf.Clean(true)
is.Equal(err, ErrUnacceptablePath) is.Equal(err, ErrUnacceptablePath)
} }

@ -2,10 +2,6 @@
port = 8002 port = 8002
host = "127.0.0.1" host = "127.0.0.1"
[admin]
port = 8003
host = "127.0.0.1"
[file] [file]
path = "/tmp/datta2" path = "/tmp/datta2"

Binary file not shown.

Before

Width:  |  Height:  |  Size: 10 KiB

@ -48,13 +48,6 @@ func (f FileSystemSwampFileData) Modified() time.Time {
return stat.ModTime() return stat.ModTime()
} }
func (f FileSystemSwampFileData) Truncate() {
err := f.file.Truncate(0)
if err != nil {
panic(err)
}
}
type Repository struct { type Repository struct {
Root string Root string
} }
@ -105,7 +98,6 @@ func (f Repository) Create(filename string, namespace_ns string) (swampfile.Swam
if err != nil { if err != nil {
panic(err) panic(err)
} }
file.Truncate(0)
bfd := FileSystemSwampFileData{filename, stat_info.Size(), stat_info.ModTime(), file} bfd := FileSystemSwampFileData{filename, stat_info.Size(), stat_info.ModTime(), file}
@ -123,13 +115,7 @@ func (f Repository) Open(filename string, namespace_ns string) (swampfile.SwampO
return nil, swampfile.ErrNotExists return nil, swampfile.ErrNotExists
} }
stat, err := file.Stat() bfd := FileSystemSwampFileData{filename, 0, time.Now(), file}
if err != nil {
return nil, err
}
bfd := FileSystemSwampFileData{filename, stat.Size(), time.Now(), file}
return bfd, nil return bfd, nil
} }

@ -11,7 +11,7 @@ type Repository struct {
NextId int64 NextId int64
} }
func NewRepository() namespace.Repository { func NewRepository() *Repository {
r := new(Repository) r := new(Repository)
r.NextId = 0 r.NextId = 0
r.IdIdx = make(map[int64]*namespace.Namespace) r.IdIdx = make(map[int64]*namespace.Namespace)

@ -2,9 +2,42 @@ package namespace
import ( import (
"caj-larsson/bog/dataswamp/namespace" "caj-larsson/bog/dataswamp/namespace"
"github.com/matryer/is"
"testing" "testing"
"time"
) )
func TestFileRepo(t *testing.T) { func TestUserAgentRepo(t *testing.T) {
namespace.RepositoryContract(NewRepository, t) is := is.New(t)
r := NewRepository()
all, err := r.All()
is.NoErr(err)
is.Equal(len(all), 0)
ns := namespace.Namespace{23, "n1", time.Now(), time.Duration(time.Hour * 3), namespace.FileSizeQuota{1000, 0}}
ns1, _ := r.Create(ns)
ns.Name = "n2"
ns2, _ := r.Create(ns)
is.True(ns1 != ns2)
all, err = r.All()
is.NoErr(err)
is.Equal(len(all), 2)
is.Equal(ns.ID, int64(23))
ns3, _ := r.GetByName("n2")
is.Equal(ns3, ns2)
is.NoErr(r.Delete(ns2.ID))
all, err = r.All()
is.NoErr(err)
is.Equal(len(all), 1)
} }

@ -44,13 +44,6 @@ func (f SwampFile) Modified() time.Time {
return stat.ModTime() return stat.ModTime()
} }
func (f SwampFile) Truncate() {
err := f.file.Truncate(0)
if err != nil {
panic(err)
}
}
// The actual repository // The actual repository
type Repository struct { type Repository struct {
Fs afero.Fs Fs afero.Fs

@ -1,31 +0,0 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.13.0
package namespace
import (
"context"
"database/sql"
)
type DBTX interface {
ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
PrepareContext(context.Context, string) (*sql.Stmt, error)
QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
QueryRowContext(context.Context, string, ...interface{}) *sql.Row
}
func New(db DBTX) *Queries {
return &Queries{db: db}
}
type Queries struct {
db DBTX
}
func (q *Queries) WithTx(tx *sql.Tx) *Queries {
return &Queries{
db: tx,
}
}

@ -1,26 +0,0 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.13.0
package namespace
import (
"database/sql"
)
type FileStat struct {
ID int64
Num int64
SizeB int64
}
type Namespace struct {
ID int64
Name string
Lastseen int64
AllowanceTime sql.NullInt64
QuotaKb sql.NullInt64
QuotaUsageKb sql.NullInt64
DownloadID int64
UploadID int64
}

@ -1,75 +0,0 @@
-- name: CreateNamespace :one
INSERT INTO
namespace(
name,
lastseen,
allowance_time,
quota_kb,
quota_usage_kb,
download_id,
upload_id
)
values(?, ?, ?, ?, ?, ?, ?)
returning id;
-- name: CreateFileStats :one
INSERT INTO file_stats(num, size_b)
values(?, ?)
returning id;
-- name: AllNamespaces :many
SELECT
ns.id,
ns.name,
ns.lastseen,
ns.allowance_time,
ns.quota_kb,
ns.quota_usage_kb,
d.num as d_num,
d.size_b as d_size_b,
ul.num as ul_num,
ul.size_b as ul_size_b
FROM namespace as ns
JOIN file_stats as d
ON ns.download_id = d.id
JOIN file_stats as ul
ON ns.upload_id = ul.id;
-- name: GetNamespaceByName :one
SELECT
ns.id,
ns.name,
ns.lastseen,
ns.allowance_time,
ns.quota_kb,
ns.quota_usage_kb,
d.num as d_num,
d.size_b as d_size_b,
ul.num as ul_num,
ul.size_b as ul_size_b
FROM namespace as ns
JOIN file_stats as d
ON ns.download_id = d.id
JOIN file_stats as ul
ON ns.upload_id = ul.id
WHERE ns.name = ?;
-- name: GetFileStat :one
SELECT * FROM file_stats where id = ?;
-- name: UpdateFileStat :exec
UPDATE file_stats SET num = ?, size_b = ? where id = ?;
-- name: UpdateNamespace :one
UPDATE namespace SET
name = ?,
lastseen = ?,
allowance_time = ?,
quota_kb = ?,
quota_usage_kb = ?
WHERE id = ?
RETURNING download_id, upload_id;
-- name: DeleteNameSpace :exec
DELETE FROM namespace where id = ?;

@ -1,260 +0,0 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.13.0
// source: queries.sql
package namespace
import (
"context"
"database/sql"
)
const allNamespaces = `-- name: AllNamespaces :many
SELECT
ns.id,
ns.name,
ns.lastseen,
ns.allowance_time,
ns.quota_kb,
ns.quota_usage_kb,
d.num as d_num,
d.size_b as d_size_b,
ul.num as ul_num,
ul.size_b as ul_size_b
FROM namespace as ns
JOIN file_stats as d
ON ns.download_id = d.id
JOIN file_stats as ul
ON ns.upload_id = ul.id
`
type AllNamespacesRow struct {
ID int64
Name string
Lastseen int64
AllowanceTime sql.NullInt64
QuotaKb sql.NullInt64
QuotaUsageKb sql.NullInt64
DNum int64
DSizeB int64
UlNum int64
UlSizeB int64
}
func (q *Queries) AllNamespaces(ctx context.Context) ([]AllNamespacesRow, error) {
rows, err := q.db.QueryContext(ctx, allNamespaces)
if err != nil {
return nil, err
}
defer rows.Close()
var items []AllNamespacesRow
for rows.Next() {
var i AllNamespacesRow
if err := rows.Scan(
&i.ID,
&i.Name,
&i.Lastseen,
&i.AllowanceTime,
&i.QuotaKb,
&i.QuotaUsageKb,
&i.DNum,
&i.DSizeB,
&i.UlNum,
&i.UlSizeB,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const createFileStats = `-- name: CreateFileStats :one
INSERT INTO file_stats(num, size_b)
values(?, ?)
returning id
`
type CreateFileStatsParams struct {
Num int64
SizeB int64
}
func (q *Queries) CreateFileStats(ctx context.Context, arg CreateFileStatsParams) (int64, error) {
row := q.db.QueryRowContext(ctx, createFileStats, arg.Num, arg.SizeB)
var id int64
err := row.Scan(&id)
return id, err
}
const createNamespace = `-- name: CreateNamespace :one
INSERT INTO
namespace(
name,
lastseen,
allowance_time,
quota_kb,
quota_usage_kb,
download_id,
upload_id
)
values(?, ?, ?, ?, ?, ?, ?)
returning id
`
type CreateNamespaceParams struct {
Name string
Lastseen int64
AllowanceTime sql.NullInt64
QuotaKb sql.NullInt64
QuotaUsageKb sql.NullInt64
DownloadID int64
UploadID int64
}
func (q *Queries) CreateNamespace(ctx context.Context, arg CreateNamespaceParams) (int64, error) {
row := q.db.QueryRowContext(ctx, createNamespace,
arg.Name,
arg.Lastseen,
arg.AllowanceTime,
arg.QuotaKb,
arg.QuotaUsageKb,
arg.DownloadID,
arg.UploadID,
)
var id int64
err := row.Scan(&id)
return id, err
}
const deleteNameSpace = `-- name: DeleteNameSpace :exec
DELETE FROM namespace where id = ?
`
func (q *Queries) DeleteNameSpace(ctx context.Context, id int64) error {
_, err := q.db.ExecContext(ctx, deleteNameSpace, id)
return err
}
const getFileStat = `-- name: GetFileStat :one
SELECT id, num, size_b FROM file_stats where id = ?
`
func (q *Queries) GetFileStat(ctx context.Context, id int64) (FileStat, error) {
row := q.db.QueryRowContext(ctx, getFileStat, id)
var i FileStat
err := row.Scan(&i.ID, &i.Num, &i.SizeB)
return i, err
}
const getNamespaceByName = `-- name: GetNamespaceByName :one
SELECT
ns.id,
ns.name,
ns.lastseen,
ns.allowance_time,
ns.quota_kb,
ns.quota_usage_kb,
d.num as d_num,
d.size_b as d_size_b,
ul.num as ul_num,
ul.size_b as ul_size_b
FROM namespace as ns
JOIN file_stats as d
ON ns.download_id = d.id
JOIN file_stats as ul
ON ns.upload_id = ul.id
WHERE ns.name = ?
`
type GetNamespaceByNameRow struct {
ID int64
Name string
Lastseen int64
AllowanceTime sql.NullInt64
QuotaKb sql.NullInt64
QuotaUsageKb sql.NullInt64
DNum int64
DSizeB int64
UlNum int64
UlSizeB int64
}
func (q *Queries) GetNamespaceByName(ctx context.Context, name string) (GetNamespaceByNameRow, error) {
row := q.db.QueryRowContext(ctx, getNamespaceByName, name)
var i GetNamespaceByNameRow
err := row.Scan(
&i.ID,
&i.Name,
&i.Lastseen,
&i.AllowanceTime,
&i.QuotaKb,
&i.QuotaUsageKb,
&i.DNum,
&i.DSizeB,
&i.UlNum,
&i.UlSizeB,
)
return i, err
}
const updateFileStat = `-- name: UpdateFileStat :exec
UPDATE file_stats SET num = ?, size_b = ? where id = ?
`
type UpdateFileStatParams struct {
Num int64
SizeB int64
ID int64
}
func (q *Queries) UpdateFileStat(ctx context.Context, arg UpdateFileStatParams) error {
_, err := q.db.ExecContext(ctx, updateFileStat, arg.Num, arg.SizeB, arg.ID)
return err
}
const updateNamespace = `-- name: UpdateNamespace :one
UPDATE namespace SET
name = ?,
lastseen = ?,
allowance_time = ?,
quota_kb = ?,
quota_usage_kb = ?
WHERE id = ?
RETURNING download_id, upload_id
`
type UpdateNamespaceParams struct {
Name string
Lastseen int64
AllowanceTime sql.NullInt64
QuotaKb sql.NullInt64
QuotaUsageKb sql.NullInt64
ID int64
}
type UpdateNamespaceRow struct {
DownloadID int64
UploadID int64
}
func (q *Queries) UpdateNamespace(ctx context.Context, arg UpdateNamespaceParams) (UpdateNamespaceRow, error) {
row := q.db.QueryRowContext(ctx, updateNamespace,
arg.Name,
arg.Lastseen,
arg.AllowanceTime,
arg.QuotaKb,
arg.QuotaUsageKb,
arg.ID,
)
var i UpdateNamespaceRow
err := row.Scan(&i.DownloadID, &i.UploadID)
return i, err
}

@ -2,188 +2,153 @@ package namespace
import ( import (
"caj-larsson/bog/dataswamp/namespace" "caj-larsson/bog/dataswamp/namespace"
"context"
"database/sql" "database/sql"
"errors"
"github.com/mattn/go-sqlite3" "github.com/mattn/go-sqlite3"
"time"
) )
var _ = sqlite3.ErrError
type Repository struct { type Repository struct {
db *sql.DB db *sql.DB
} }
func (r *Repository) migrate() error { func NewRepository(filename string) *Repository {
query := `
CREATE TABLE IF NOT EXISTS file_stats(
id INTEGER PRIMARY KEY,
num BIGINT NOT NULL,
size_b BIGINT NOT NULL
);`
_, err := r.db.Exec(query)
if err != nil {
return err
}
query = `
CREATE TABLE IF NOT EXISTS namespace(
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
lastseen TEXT,
allowance_time BIGINT,
quota_kb BIGINT,
quota_usage_kb BIGINT,
download_id BIGINT NOT NULL REFERENCES file_stats(Id),
upload_id BIGINT NOT NULL REFERENCES file_stats(Id)
);`
_, err = r.db.Exec(query)
return err
}
func NewRepository(filename string) namespace.Repository {
db, err := sql.Open("sqlite3", filename) db, err := sql.Open("sqlite3", filename)
if err != nil { if err != nil {
panic(err) panic(err)
} }
repo := Repository{ repo := Repository{
db: db, db: db,
} }
repo.migrate() repo.migrate()
return &repo return &repo
} }
func (q *Queries) createFileStats(ctx context.Context, fstat namespace.FileStat) (int64, error) { func (r Repository) migrate() error {
return q.CreateFileStats(ctx, CreateFileStatsParams{fstat.Num, fstat.SizeB}) query := `
CREATE TABLE IF NOT EXISTS namespace(
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
lastseen text,
allowance_time bigint,
quota_kb bigint,
quota_usage_kb bigint
);
`
_, err := r.db.Exec(query)
return err
} }
func (r *Repository) Create(ns namespace.Namespace) (*namespace.Namespace, error) { func (r *Repository) Create(ns namespace.Namespace) (*namespace.Namespace, error) {
ctx := context.Background() var record, err = fromEntity(ns)
q := New(r.db)
dl_id, err := q.createFileStats(ctx, ns.Download)
if err != nil { if err != nil {
return nil, err
}
ul_id, err := q.createFileStats(ctx, ns.Upload) }
res, err := r.db.Exec(
"INSERT INTO namespace(name, lastseen, allowance_time, quota_kb, quota_usage_kb) values(?,?,?,?,?)",
record.Name, record.LastSeen, record.AllowanceSeconds, record.QuotaKB, record.QuotaUsedKB,
)
if err != nil { if err != nil {
return nil, err var sqliteErr sqlite3.Error
if errors.As(err, &sqliteErr) {
if errors.Is(sqliteErr.ExtendedCode, sqlite3.ErrConstraintUnique) {
return nil, namespace.ErrDuplicate
} }
}
ns.LastSeen = ns.LastSeen.Round(time.Microsecond) return nil, err
p := CreateNamespaceParams{
Name: ns.Name,
Lastseen: ns.LastSeen.UnixMicro(),
AllowanceTime: sql.NullInt64{int64(ns.AllowanceDuration.Seconds()), true},
QuotaKb: sql.NullInt64{int64(ns.FileQuota.AllowanceKB), true},
QuotaUsageKb: sql.NullInt64{int64(ns.FileQuota.CurrentUsage), true},
DownloadID: dl_id,
UploadID: ul_id,
} }
id, err := q.CreateNamespace(ctx, p) id, err := res.LastInsertId()
if err != nil { if err != nil {
return nil, err return nil, err
} }
ns.ID = id ns.ID = id
return &ns, nil return &ns, nil
} }
func (r *Repository) All() ([]namespace.Namespace, error) { func (r Repository) All() ([]namespace.Namespace, error) {
ctx := context.Background() rows, err := r.db.Query("SELECT * FROM namespace")
q := New(r.db)
rows, err := q.AllNamespaces(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer rows.Close()
var all []namespace.Namespace var all []namespace.Namespace
for _, row := range rows { for rows.Next() {
ns := namespace.Namespace{ var record NamespaceRecord
row.ID, if err := rows.Scan(&record.ID, &record.Name, &record.LastSeen, &record.AllowanceSeconds, &record.QuotaKB, &record.QuotaUsedKB); err != nil {
row.Name, return nil, err
time.UnixMicro(row.Lastseen),
time.Duration(row.AllowanceTime.Int64 * int64(time.Second)),
namespace.FileSizeQuota{row.QuotaKb.Int64, row.QuotaUsageKb.Int64},
namespace.FileStat{row.DNum, row.DSizeB},
namespace.FileStat{row.UlNum, row.UlSizeB},
} }
var ns, err = record.toEntity()
all = append(all, ns) if err != nil {
return nil, err
}
all = append(all, *ns)
} }
return all, nil return all, nil
} }
func (r *Repository) GetByName(name string) (*namespace.Namespace, error) { func (r Repository) GetByName(name string) (*namespace.Namespace, error) {
ctx := context.Background() row := r.db.QueryRow("SELECT id, name, lastseen, allowance_time, quota_kb, quota_usage_kb FROM namespace WHERE name = ?", name)
q := New(r.db)
row, err := q.GetNamespaceByName(ctx, name)
if err != nil { var record NamespaceRecord
if err := row.Scan(&record.ID, &record.Name, &record.LastSeen, &record.AllowanceSeconds, &record.QuotaKB, &record.QuotaUsedKB); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, namespace.ErrNotExists return nil, namespace.ErrNotExists
} }
return nil, err
ns := namespace.Namespace{
row.ID,
row.Name,
time.UnixMicro(row.Lastseen),
time.Duration(row.AllowanceTime.Int64 * int64(time.Second)),
namespace.FileSizeQuota{row.QuotaKb.Int64, row.QuotaUsageKb.Int64},
namespace.FileStat{row.DNum, row.DSizeB},
namespace.FileStat{row.UlNum, row.UlSizeB},
} }
return &ns, nil var ns, err = record.toEntity()
if err != nil {
return nil, err
}
return ns, nil
} }
func (q *Queries) updateFileStat(ctx context.Context, id int64, fstat namespace.FileStat) error { func (r Repository) Update(id int64, updated namespace.Namespace) (*namespace.Namespace, error) {
return q.UpdateFileStat(ctx, UpdateFileStatParams{fstat.Num, fstat.SizeB, id}) if id == 0 {
return nil, errors.New("invalid updated ID")
} }
func (r *Repository) Update(id int64, ns namespace.Namespace) (*namespace.Namespace, error) { var record, err = fromEntity(updated)
ctx := context.Background()
q := New(r.db)
ids, err := q.UpdateNamespace(ctx, UpdateNamespaceParams{ res, err := r.db.Exec("UPDATE namespace SET name = ?, lastseen = ?, allowance_time = ?, quota_kb = ?, quota_usage_kb = ? WHERE id = ?",
ns.Name, record.Name, record.LastSeen, record.AllowanceSeconds, &record.QuotaKB, &record.QuotaUsedKB, id)
ns.LastSeen.Round(time.Microsecond).UnixMicro(),
sql.NullInt64{int64(ns.AllowanceDuration.Seconds()), true},
sql.NullInt64{int64(ns.FileQuota.AllowanceKB), true},
sql.NullInt64{int64(ns.FileQuota.CurrentUsage), true},
ns.ID,
})
err = q.updateFileStat(ctx, ids.DownloadID, ns.Download)
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = q.updateFileStat(ctx, ids.UploadID, ns.Upload) rowsAffected, err := res.RowsAffected()
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &ns, nil if rowsAffected == 0 {
return nil, namespace.ErrUpdateFailed
}
return &updated, nil
} }
func (r *Repository) Delete(id int64) error { func (r Repository) Delete(id int64) error {
ctx := context.Background() res, err := r.db.Exec("DELETE FROM namespace WHERE id = ?", id)
q := New(r.db) if err != nil {
return err
}
err := q.DeleteNameSpace(ctx, id) rowsAffected, err := res.RowsAffected()
if err != nil { if err != nil {
return err
}
if rowsAffected == 0 {
return namespace.ErrDeleteFailed return namespace.ErrDeleteFailed
} }
return nil return err
} }

@ -1,10 +0,0 @@
package namespace
import (
"caj-larsson/bog/dataswamp/namespace"
"testing"
)
func TestFileRepo(t *testing.T) {
namespace.RepositoryContract(func() namespace.Repository { return NewRepository(":memory:") }, t)
}

@ -1,16 +0,0 @@
CREATE TABLE IF NOT EXISTS file_stats(
id BIGINT PRIMARY KEY,
num BIGINT NOT NULL,
size_b BIGINT NOT NULL
);
CREATE TABLE IF NOT EXISTS namespace(
id BIGINT PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
lastseen BIGINT NOT NULL,
allowance_time BIGINT,
quota_kb BIGINT,
quota_usage_kb BIGINT,
download_id BIGINT NOT NULL REFERENCES file_stats(Id) ON DELETE CASCADE,
upload_id BIGINT NOT NULL REFERENCES file_stats(Id) ON DELETE CASCADE
);

@ -1,11 +0,0 @@
package system_time
import (
"time"
)
type Clock struct{}
func (c Clock) Now() time.Time {
return time.Now()
}

@ -2,6 +2,7 @@ package main
import ( import (
"caj-larsson/bog/server" "caj-larsson/bog/server"
"fmt"
"io/ioutil" "io/ioutil"
) )
@ -16,6 +17,9 @@ func main() {
if err != nil { if err != nil {
panic(err) panic(err)
} }
fmt.Printf("Dataswamp running")
bog := server.New(config) bog := server.New(config)
bog.Run() bog.Run()
} }

@ -1,17 +1,15 @@
package server package server
import ( import (
"fmt"
"net/http"
"strconv"
"time"
"caj-larsson/bog/dataswamp" "caj-larsson/bog/dataswamp"
"caj-larsson/bog/dataswamp/namespace" "caj-larsson/bog/dataswamp/namespace"
"caj-larsson/bog/dataswamp/swampfile" "caj-larsson/bog/dataswamp/swampfile"
fs_swampfile "caj-larsson/bog/infrastructure/fs/swampfile" fs_swampfile "caj-larsson/bog/infrastructure/fs/swampfile"
sql_namespace "caj-larsson/bog/infrastructure/sqlite/namespace" sql_namespace "caj-larsson/bog/infrastructure/sqlite/namespace"
"caj-larsson/bog/infrastructure/system_time"
"caj-larsson/bog/util"
"net/http"
"strconv"
"text/template"
"time"
) )
type Router interface { type Router interface {
@ -21,11 +19,8 @@ type Router interface {
type Bog struct { type Bog struct {
router Router router Router
adminRouter Router file_service dataswamp.SwampFileService
file_service dataswamp.DataSwampService
address string address string
adminAddress string
logger util.Logger
} }
func buildFileDataRepository(config FileConfig) swampfile.Repository { func buildFileDataRepository(config FileConfig) swampfile.Repository {
@ -44,39 +39,26 @@ func buildNamespaceRepository(config DatabaseConfig) namespace.Repository {
} }
func (b *Bog) fileHandler(w http.ResponseWriter, r *http.Request) { func (b *Bog) fileHandler(w http.ResponseWriter, r *http.Request) {
ref := swampfile.FileReference{r.URL.Path, r.Header["User-Agent"][0]}
if r.URL.Path == "/" { if r.URL.Path == "/" {
http.NotFound(w, r) fmt.Fprintf(w, "Hi")
return return
} }
ref := swampfile.FileReference{r.URL.Path, r.Header["User-Agent"][0]}
switch r.Method { switch r.Method {
case "GET": case "GET":
swamp_file, err := b.file_service.OpenOutFile(ref) swamp_file, err := b.file_service.OpenOutFile(ref)
if err == swampfile.ErrNotExists { if err == swampfile.ErrNotExists {
templ, err := template.ParseFiles("server/views/upload.html") http.NotFound(w, r)
if err != nil {
panic(err)
}
w.WriteHeader(404)
err = templ.Execute(w, nil)
if err != nil {
panic(err)
}
return return
} }
if err != nil { if err != nil {
panic(err) panic(err)
} }
b.logger.Info("Serving file '%s' of size %d from '%s'", ref.Path, swamp_file.Size(), ref.UserAgent)
w.Header().Set("Content-Disposition", "attachment")
http.ServeContent(w, r, swamp_file.Path(), swamp_file.Modified(), swamp_file) http.ServeContent(w, r, swamp_file.Path(), swamp_file.Modified(), swamp_file)
case "POST": case "POST":
fallthrough fallthrough
@ -89,7 +71,6 @@ func (b *Bog) fileHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
b.logger.Info("Recieving file '%s' of size %d from '%s'", ref.Path, size, ref.UserAgent)
err = b.file_service.SaveFile(ref, r.Body, size) err = b.file_service.SaveFile(ref, r.Body, size)
if err != nil { if err != nil {
@ -99,24 +80,8 @@ func (b *Bog) fileHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
func (b *Bog) dashboardHandler(w http.ResponseWriter, r *http.Request) {
templ, err := template.ParseFiles("server/views/dashboard.html")
if err != nil {
panic(err)
}
stats := b.file_service.NamespaceStats()
err = templ.Execute(w, stats)
if err != nil {
panic(err)
}
}
func (b *Bog) routes() { func (b *Bog) routes() {
b.router.HandleFunc("/", b.fileHandler) b.router.HandleFunc("/", b.fileHandler)
// b.adminRouter.HandleFunc("/", b.dashboardHandler)
} }
func (b *Bog) cleanNamespaces(){ func (b *Bog) cleanNamespaces(){
@ -128,36 +93,21 @@ func (b *Bog) cleanNamespaces() {
func New(config *Configuration) *Bog { func New(config *Configuration) *Bog {
b := new(Bog) b := new(Bog)
b.address = config.Server.bindAddress() b.address = config.bindAddress()
b.adminAddress = config.Admin.bindAddress()
fsSwampRepo := buildFileDataRepository(config.File) fsSwampRepo := buildFileDataRepository(config.File)
nsRepo := buildNamespaceRepository(config.Database) nsRepo := buildNamespaceRepository(config.Database)
logger := ServerLogger{Debug} b.file_service = dataswamp.NewSwampFileService(
ns_svc := namespace.NewNamespaceService( nsRepo, fsSwampRepo, config.Quota.ParsedSizeBytes(), config.Quota.ParsedDuration(),
nsRepo,
logger,
system_time.Clock{},
config.Quota.ParsedDuration(),
config.Quota.ParsedSizeBytes(),
) )
b.file_service = *dataswamp.NewDataSwampService(
*ns_svc,
fsSwampRepo,
logger,
)
b.logger = logger
b.router = new(http.ServeMux) b.router = new(http.ServeMux)
b.adminRouter = new(http.ServeMux)
b.routes() b.routes()
return b return b
} }
func (b *Bog) Run() { func (b *Bog) Run() {
b.logger.Info("Starting bog on address: %s", b.address)
go b.cleanNamespaces() go b.cleanNamespaces()
go func() { http.ListenAndServe(b.adminAddress, b.adminRouter) }()
http.ListenAndServe(b.address, b.router) http.ListenAndServe(b.address, b.router)
} }

@ -4,10 +4,8 @@ import (
"testing" "testing"
// "fmt" // "fmt"
"caj-larsson/bog/dataswamp" "caj-larsson/bog/dataswamp"
"caj-larsson/bog/dataswamp/namespace" "caj-larsson/bog/infrastructure/memory/namespace"
ns "caj-larsson/bog/infrastructure/memory/namespace"
"caj-larsson/bog/infrastructure/memory/swampfile" "caj-larsson/bog/infrastructure/memory/swampfile"
"caj-larsson/bog/infrastructure/system_time"
"github.com/matryer/is" "github.com/matryer/is"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@ -15,36 +13,19 @@ import (
"time" "time"
) )
type TestLogger struct{}
func (t TestLogger) Debug(format string, a ...interface{}) {}
func (t TestLogger) Info(format string, a ...interface{}) {}
func (t TestLogger) Warn(format string, a ...interface{}) {}
func TestApplication(t *testing.T) { func TestApplication(t *testing.T) {
is := is.New(t) is := is.New(t)
logger := TestLogger{} file_service := dataswamp.NewSwampFileService(
nsRepo := ns.NewRepository() namespace.NewRepository(),
ns_svc := namespace.NewNamespaceService(
nsRepo,
logger,
system_time.Clock{},
time.Hour,
1000,
)
file_service := dataswamp.NewDataSwampService(
*ns_svc,
swampfile.NewRepository(), swampfile.NewRepository(),
logger, 1000,
time.Hour,
) )
bog := Bog{ bog := Bog{
router: new(http.ServeMux), router: new(http.ServeMux),
file_service: *file_service, file_service: file_service,
address: "fake", address: "fake",
logger: logger,
} }
bog.routes() bog.routes()
req := httptest.NewRequest("POST", "/apath", strings.NewReader("testdata")) req := httptest.NewRequest("POST", "/apath", strings.NewReader("testdata"))

@ -54,21 +54,15 @@ type DatabaseConfig struct {
Connection string Connection string
} }
type LoggingConfig struct {
Level string
}
type Configuration struct { type Configuration struct {
Server ServerConfig Server ServerConfig
Admin ServerConfig
File FileConfig File FileConfig
Database DatabaseConfig Database DatabaseConfig
Quota QuotaConfig Quota QuotaConfig
Logging LoggingConfig
} }
func (c ServerConfig) bindAddress() string { func (c *Configuration) bindAddress() string {
return fmt.Sprintf("%s:%d", c.Host, c.Port) return fmt.Sprintf("%s:%d", c.Server.Host, c.Server.Port)
} }
func ConfigFromToml(toml_data string) (*Configuration, error) { func ConfigFromToml(toml_data string) (*Configuration, error) {

@ -8,15 +8,11 @@ import (
func TestConfiguration(t *testing.T) { func TestConfiguration(t *testing.T) {
is := is.New(t) is := is.New(t)
c, _ := ConfigFromToml(` c, _ := ConfigFromToml(
[server] `[server]
port = 8002 port = 8002
host = "127.0.0.1" host = "127.0.0.1"
[admin]
port = 8001
host = "127.0.0.1"
[file] [file]
path = "/tmp/datta2" path = "/tmp/datta2"

@ -1,40 +0,0 @@
package server
import (
"fmt"
"time"
)
const (
Debug int = 0
Info = 1
Warn = 2
None = 3
)
type ServerLogger struct {
level int
}
func logf(level string, format string, a ...interface{}) {
head := fmt.Sprintf("%s - [%s]: ", time.Now().Format(time.RFC3339), level)
fmt.Printf(head+format+"\n", a...)
}
func (t ServerLogger) Debug(format string, a ...interface{}) {
if t.level <= Debug {
logf("DEBUG", format, a...)
}
}
func (t ServerLogger) Info(format string, a ...interface{}) {
if t.level <= Info {
logf("INFO", format, a...)
}
}
func (t ServerLogger) Warn(format string, a ...interface{}) {
if t.level <= Warn {
logf("WARN", format, a...)
}
}

@ -1,33 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>bog</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3" crossorigin="anonymous">
</head>
<body>
<table class="table">
<thead>
<tr>
<th scope="col">Name</th>
<th scope="col">Last Seen</th>
<th scope="col">Quota</th>
<th scope="col">Usage</th>
<th scope="col">Download</th>
<th scope="col">Upload</th>
</tr>
</thead>
<tbody>
{{ range . }}
<tr>
<td>{{ .Name }}</td>
<td>{{ .LastSeen }} - {{ .AllowanceDuration }}</td>
<td>{{ .FileQuota.CurrentUsage }}/{{ .FileQuota.AllowanceKB}} B</td>
<td>{{ .Usage.SizeB }}B - {{ .Usage.Num }} </td>
<td>{{ .Download.SizeB }}B - {{ .Download.Num }} </td>
<td>{{ .Upload.SizeB }}B - {{ .Upload.Num }} </td>
</tr>
{{ end }}
</tbody>
</table>
</body>
</html>

@ -1,25 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>bog</title>
</head>
<body>
<script>
function upload() {
var file = document.getElementById('file').files[0];
var ajax = new XMLHttpRequest;
const reader = new FileReader();
ajax.open('POST', '', true);
ajax.overrideMimeType('text/plain; charset=x-user-defined-binary');
reader.onload = function(evt) {
ajax.send(evt.target.result);
};
reader.readAsBinaryString(file);
}
</script>
<input id="file" type="file" />
<input type="button" onClick="upload()" value="Upload">
</body>
</html>

@ -1,8 +0,0 @@
version: "1"
packages:
- path: infrastructure/sqlite/namespace
name: namespace
engine: postgresql
schema: infrastructure/sqlite/namespace/schema.sql
queries: infrastructure/sqlite/namespace/queries.sql

@ -1,41 +0,0 @@
package util
type Event struct {
eventName string
payload interface{}
}
func NewEvent(eventName string, payload interface{}) *Event {
return &Event{
eventName,
payload,
}
}
func (e *Event) EventName() string {
return e.eventName
}
func (e *Event) Payload() interface{} {
return e.payload
}
type EventHandler func(payload interface{})
type EventBus struct {
handlers map[string][]EventHandler
}
func NewEventBus() *EventBus {
return &EventBus{make(map[string][]EventHandler)}
}
func (eb *EventBus) Register(eventName string, handler EventHandler) {
eb.handlers[eventName] = append(eb.handlers[eventName], handler)
}
func (eb *EventBus) Handle(e Event) {
for _, handler := range eb.handlers[e.EventName()] {
handler(e.Payload())
}
}

@ -1,23 +0,0 @@
package util
import (
"github.com/matryer/is"
"testing"
)
func (eb *EventBus) Handled(e Event) bool {
// TODO: figure out how to verify the event signature here.
handlers, exists := eb.handlers[e.EventName()]
if !exists {
return false
}
return len(handlers) > 0
}
func AcceptsMessage(t *testing.T, eb *EventBus, es []Event) {
is := is.New(t)
for _, e := range es {
is.True(eb.Handled(e))
}
}

@ -1,13 +0,0 @@
package util
type Logger interface {
Debug(format string, a ...interface{})
Info(format string, a ...interface{})
Warn(format string, a ...interface{})
}
type TestLogger struct{}
func (t TestLogger) Debug(format string, a ...interface{}) {}
func (t TestLogger) Info(format string, a ...interface{}) {}
func (t TestLogger) Warn(format string, a ...interface{}) {}
Loading…
Cancel
Save