diff --git a/cmd/api/main.go b/cmd/api/main.go index 84895ee..8471725 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -122,19 +122,13 @@ func main() { Msg("Email service not configured - emails will not be sent") } - // Initialize storage service for file uploads + // Initialize storage service for file uploads (local filesystem or S3-compatible) var storageService *services.StorageService - if cfg.Storage.UploadDir != "" { + if cfg.Storage.UploadDir != "" || cfg.Storage.IsS3() { storageService, err = services.NewStorageService(&cfg.Storage) if err != nil { log.Warn().Err(err).Msg("Failed to initialize storage service - uploads disabled") } else { - log.Info(). - Str("upload_dir", cfg.Storage.UploadDir). - Str("base_url", cfg.Storage.BaseURL). - Int64("max_file_size", cfg.Storage.MaxFileSize). - Msg("Storage service initialized") - // Initialize file encryption at rest if configured if cfg.Storage.EncryptionKey != "" { encSvc, encErr := services.NewEncryptionService(cfg.Storage.EncryptionKey) diff --git a/go.mod b/go.mod index a71c349..798f110 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/treytartt/honeydue-api -go 1.24.0 +go 1.25 require ( github.com/go-pdf/fpdf v0.9.0 @@ -10,6 +10,7 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/hibiken/asynq v0.25.1 github.com/labstack/echo/v4 v4.11.4 + github.com/minio/minio-go/v7 v7.0.99 github.com/nicksnyder/go-i18n/v2 v2.6.0 github.com/redis/go-redis/v9 v9.17.1 github.com/rs/zerolog v1.34.0 @@ -20,9 +21,9 @@ require ( github.com/stretchr/testify v1.11.1 github.com/stripe/stripe-go/v81 v81.4.0 github.com/wneessen/go-mail v0.7.2 - golang.org/x/crypto v0.45.0 + golang.org/x/crypto v0.46.0 golang.org/x/oauth2 v0.34.0 - golang.org/x/text v0.31.0 + golang.org/x/text v0.32.0 golang.org/x/time v0.14.0 google.golang.org/api v0.257.0 gopkg.in/yaml.v3 v3.0.1 @@ -31,6 +32,20 @@ require ( gorm.io/gorm v1.31.1 ) +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect +) + require ( cloud.google.com/go/auth v0.17.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect @@ -85,9 +100,9 @@ require ( go.opentelemetry.io/otel v1.38.0 // indirect go.opentelemetry.io/otel/metric v1.38.0 // indirect go.opentelemetry.io/otel/trace v1.38.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 // indirect google.golang.org/grpc v1.77.0 // indirect google.golang.org/protobuf v1.36.10 // indirect diff --git a/go.sum b/go.sum index b3bb2e0..7a073b0 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -28,6 +30,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -84,6 +88,13 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -104,10 +115,18 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= github.com/nicksnyder/go-i18n/v2 v2.6.0 h1:C/m2NNWNiTB6SK4Ao8df5EWm3JETSTIGNXBpMJTxzxQ= github.com/nicksnyder/go-i18n/v2 v2.6.0/go.mod h1:88sRqr0C6OPyJn0/KRNaEz1uWorjxIKP7rUUcvycecE= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -119,6 +138,7 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= @@ -154,6 +174,8 @@ github.com/stripe/stripe-go/v81 v81.4.0 h1:AuD9XzdAvl193qUCSaLocf8H+nRopOouXhxqJ github.com/stripe/stripe-go/v81 v81.4.0/go.mod h1:C/F4jlmnGNacvYtBp/LUHCvVUJEZffFQCobkzwY1WOo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -182,17 +204,19 @@ go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJr go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20170512130425-ab89591268e0/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220403103023-749bd193bc2b/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -204,14 +228,14 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/config/config.go b/internal/config/config.go index 148f63f..dcd7937 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -138,15 +138,33 @@ type SecurityConfig struct { TokenRefreshDays int // Token must be at least this many days old before refresh (default 60) } -// StorageConfig holds file storage settings +// StorageConfig holds file storage settings. +// When S3Endpoint is set, files are stored in S3-compatible storage (B2, MinIO). +// When S3Endpoint is empty, files are stored on the local filesystem using UploadDir. type StorageConfig struct { - UploadDir string // Directory to store uploaded files - BaseURL string // Public URL prefix for serving files (e.g., "/uploads") + // Local filesystem settings + UploadDir string // Directory to store uploaded files (local mode) + BaseURL string // Public URL prefix for serving files (e.g., "/uploads") + + // S3-compatible storage settings (B2, MinIO) + S3Endpoint string // S3 endpoint (e.g., "s3.us-west-004.backblazeb2.com" or "minio:9000") + S3KeyID string // Access key ID + S3AppKey string // Secret access key + S3Bucket string // Bucket name + S3UseSSL bool // Use HTTPS (true for B2, false for in-cluster MinIO) + S3Region string // Region (optional, defaults to "us-east-1") + + // Shared settings MaxFileSize int64 // Max file size in bytes (default 10MB) AllowedTypes string // Comma-separated MIME types EncryptionKey string // 64-char hex key for file encryption at rest (optional) } +// IsS3 returns true if S3-compatible storage is configured +func (c *StorageConfig) IsS3() bool { + return c.S3Endpoint != "" && c.S3KeyID != "" && c.S3AppKey != "" && c.S3Bucket != "" +} + // FeatureFlags holds kill switches for major subsystems. // All default to true (enabled). Set to false via env vars to disable. type FeatureFlags struct { @@ -270,6 +288,12 @@ func Load() (*Config, error) { Storage: StorageConfig{ UploadDir: viper.GetString("STORAGE_UPLOAD_DIR"), BaseURL: viper.GetString("STORAGE_BASE_URL"), + S3Endpoint: viper.GetString("B2_ENDPOINT"), + S3KeyID: viper.GetString("B2_KEY_ID"), + S3AppKey: viper.GetString("B2_APP_KEY"), + S3Bucket: viper.GetString("B2_BUCKET_NAME"), + S3UseSSL: viper.GetString("STORAGE_USE_SSL") == "" || viper.GetBool("STORAGE_USE_SSL"), + S3Region: viper.GetString("B2_REGION"), MaxFileSize: viper.GetInt64("STORAGE_MAX_FILE_SIZE"), AllowedTypes: viper.GetString("STORAGE_ALLOWED_TYPES"), EncryptionKey: viper.GetString("STORAGE_ENCRYPTION_KEY"), diff --git a/internal/services/storage_backend.go b/internal/services/storage_backend.go new file mode 100644 index 0000000..617584b --- /dev/null +++ b/internal/services/storage_backend.go @@ -0,0 +1,21 @@ +package services + +import "io" + +// StorageBackend abstracts where files are physically stored. +// The StorageService handles validation, encryption, and URL generation, +// then delegates raw I/O to the backend. +type StorageBackend interface { + // Write stores data at the given key (e.g., "images/20240101_uuid.jpg"). + Write(key string, data []byte) error + + // Read returns the raw bytes stored at the given key. + Read(key string) ([]byte, error) + + // Delete removes the object at the given key. Returns nil if not found. + Delete(key string) error + + // ReadStream returns a reader for the object (used for large files). + // Callers must close the returned ReadCloser. + ReadStream(key string) (io.ReadCloser, error) +} diff --git a/internal/services/storage_backend_local.go b/internal/services/storage_backend_local.go new file mode 100644 index 0000000..1e2735e --- /dev/null +++ b/internal/services/storage_backend_local.go @@ -0,0 +1,70 @@ +package services + +import ( + "fmt" + "io" + "os" + "path/filepath" +) + +// LocalBackend stores files on the local filesystem. +type LocalBackend struct { + baseDir string +} + +// NewLocalBackend creates a local filesystem storage backend. +// It ensures the base directory and standard subdirectories exist. +func NewLocalBackend(baseDir string) (*LocalBackend, error) { + if err := os.MkdirAll(baseDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create upload directory: %w", err) + } + + for _, subdir := range []string{"images", "documents", "completions"} { + path := filepath.Join(baseDir, subdir) + if err := os.MkdirAll(path, 0755); err != nil { + return nil, fmt.Errorf("failed to create subdirectory %s: %w", subdir, err) + } + } + + return &LocalBackend{baseDir: baseDir}, nil +} + +func (b *LocalBackend) Write(key string, data []byte) error { + destPath, err := SafeResolvePath(b.baseDir, key) + if err != nil { + return fmt.Errorf("invalid path: %w", err) + } + return os.WriteFile(destPath, data, 0644) +} + +func (b *LocalBackend) Read(key string) ([]byte, error) { + fullPath, err := SafeResolvePath(b.baseDir, key) + if err != nil { + return nil, fmt.Errorf("invalid path: %w", err) + } + return os.ReadFile(fullPath) +} + +func (b *LocalBackend) Delete(key string) error { + fullPath, err := SafeResolvePath(b.baseDir, key) + if err != nil { + return nil // invalid path = nothing to delete + } + if err := os.Remove(fullPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to delete file: %w", err) + } + return nil +} + +func (b *LocalBackend) ReadStream(key string) (io.ReadCloser, error) { + fullPath, err := SafeResolvePath(b.baseDir, key) + if err != nil { + return nil, fmt.Errorf("invalid path: %w", err) + } + return os.Open(fullPath) +} + +// BaseDir returns the local storage base directory. +func (b *LocalBackend) BaseDir() string { + return b.baseDir +} diff --git a/internal/services/storage_backend_s3.go b/internal/services/storage_backend_s3.go new file mode 100644 index 0000000..572d49c --- /dev/null +++ b/internal/services/storage_backend_s3.go @@ -0,0 +1,103 @@ +package services + +import ( + "bytes" + "context" + "fmt" + "io" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/rs/zerolog/log" +) + +// S3Backend stores files in S3-compatible storage (Backblaze B2, MinIO, AWS S3). +type S3Backend struct { + client *minio.Client + bucket string +} + +// NewS3Backend creates an S3-compatible storage backend. +func NewS3Backend(endpoint, keyID, appKey, bucket string, useSSL bool, region string) (*S3Backend, error) { + if region == "" { + region = "us-east-1" + } + + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(keyID, appKey, ""), + Secure: useSSL, + Region: region, + }) + if err != nil { + return nil, fmt.Errorf("failed to create S3 client: %w", err) + } + + // Verify bucket exists + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + exists, err := client.BucketExists(ctx, bucket) + if err != nil { + return nil, fmt.Errorf("failed to check bucket %q: %w", bucket, err) + } + if !exists { + return nil, fmt.Errorf("bucket %q does not exist", bucket) + } + + log.Info(). + Str("endpoint", endpoint). + Str("bucket", bucket). + Bool("ssl", useSSL). + Msg("S3 storage backend initialized") + + return &S3Backend{client: client, bucket: bucket}, nil +} + +func (b *S3Backend) Write(key string, data []byte) error { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + _, err := b.client.PutObject(ctx, b.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}) + if err != nil { + return fmt.Errorf("failed to upload to S3: %w", err) + } + return nil +} + +func (b *S3Backend) Read(key string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + obj, err := b.client.GetObject(ctx, b.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get S3 object: %w", err) + } + defer obj.Close() + + data, err := io.ReadAll(obj) + if err != nil { + return nil, fmt.Errorf("failed to read S3 object: %w", err) + } + return data, nil +} + +func (b *S3Backend) Delete(key string) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + err := b.client.RemoveObject(ctx, b.bucket, key, minio.RemoveObjectOptions{}) + if err != nil { + return fmt.Errorf("failed to delete S3 object: %w", err) + } + return nil +} + +func (b *S3Backend) ReadStream(key string) (io.ReadCloser, error) { + ctx := context.Background() // caller controls lifetime by closing the reader + obj, err := b.client.GetObject(ctx, b.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get S3 object stream: %w", err) + } + return obj, nil +} diff --git a/internal/services/storage_service.go b/internal/services/storage_service.go index 971acba..b6f1fc6 100644 --- a/internal/services/storage_service.go +++ b/internal/services/storage_service.go @@ -5,7 +5,6 @@ import ( "io" "mime/multipart" "net/http" - "os" "path/filepath" "strings" "time" @@ -16,9 +15,11 @@ import ( "github.com/treytartt/honeydue-api/internal/config" ) -// StorageService handles file uploads to local filesystem +// StorageService handles file uploads, validation, encryption, and URL generation. +// It delegates raw I/O to a StorageBackend (local filesystem or S3-compatible). type StorageService struct { cfg *config.StorageConfig + backend StorageBackend allowedTypes map[string]struct{} // P-12: Parsed once at init for O(1) lookups encryptionSvc *EncryptionService } @@ -31,37 +32,40 @@ type UploadResult struct { MimeType string `json:"mime_type"` } -// NewStorageService creates a new storage service +// NewStorageService creates a new storage service with the appropriate backend. +// If S3 config is set, uses S3-compatible storage (B2, MinIO). +// Otherwise, uses local filesystem. func NewStorageService(cfg *config.StorageConfig) (*StorageService, error) { - // Ensure upload directory exists - if err := os.MkdirAll(cfg.UploadDir, 0755); err != nil { - return nil, fmt.Errorf("failed to create upload directory: %w", err) - } + var backend StorageBackend + var err error - // Create subdirectories for organization - subdirs := []string{"images", "documents", "completions"} - for _, subdir := range subdirs { - path := filepath.Join(cfg.UploadDir, subdir) - if err := os.MkdirAll(path, 0755); err != nil { - return nil, fmt.Errorf("failed to create subdirectory %s: %w", subdir, err) + if cfg.IsS3() { + backend, err = NewS3Backend(cfg.S3Endpoint, cfg.S3KeyID, cfg.S3AppKey, cfg.S3Bucket, cfg.S3UseSSL, cfg.S3Region) + if err != nil { + return nil, fmt.Errorf("failed to initialize S3 storage: %w", err) } + log.Info(). + Str("endpoint", cfg.S3Endpoint). + Str("bucket", cfg.S3Bucket). + Bool("ssl", cfg.S3UseSSL). + Msg("Storage service initialized (S3)") + } else { + backend, err = NewLocalBackend(cfg.UploadDir) + if err != nil { + return nil, fmt.Errorf("failed to initialize local storage: %w", err) + } + log.Info(). + Str("upload_dir", cfg.UploadDir). + Msg("Storage service initialized (local)") } // P-12: Parse AllowedTypes once at initialization for O(1) lookups - allowedTypes := make(map[string]struct{}) - for _, t := range strings.Split(cfg.AllowedTypes, ",") { - trimmed := strings.TrimSpace(t) - if trimmed != "" { - allowedTypes[trimmed] = struct{}{} - } - } + allowedTypes := parseAllowedTypes(cfg.AllowedTypes) - log.Info().Str("upload_dir", cfg.UploadDir).Int("allowed_types", len(allowedTypes)).Msg("Storage service initialized") - - return &StorageService{cfg: cfg, allowedTypes: allowedTypes}, nil + return &StorageService{cfg: cfg, backend: backend, allowedTypes: allowedTypes}, nil } -// Upload saves a file to the local filesystem +// Upload saves a file to storage (local or S3) func (s *StorageService) Upload(file *multipart.FileHeader, category string) (*UploadResult, error) { // Validate file size if file.Size > s.cfg.MaxFileSize { @@ -90,13 +94,10 @@ func (s *StorageService) Upload(file *multipart.FileHeader, category string) (*U detectedMimeType := http.DetectContentType(sniffBuf[:n]) // Validate that the detected type matches the claimed type (at the category level) - // Allow application/octet-stream from detection since DetectContentType may not - // recognize all valid types, but the claimed type must still be in our allowed list if detectedMimeType != "application/octet-stream" && !s.mimeTypesCompatible(claimedMimeType, detectedMimeType) { return nil, fmt.Errorf("file content type mismatch: claimed %s but detected %s", claimedMimeType, detectedMimeType) } - // Use the claimed MIME type (which is more specific) if it's allowed mimeType := claimedMimeType // Validate MIME type against allowed list @@ -131,11 +132,8 @@ func (s *StorageService) Upload(file *multipart.FileHeader, category string) (*U storedFilename = newFilename + ".enc" } - // S-18: Sanitize path to prevent traversal attacks - destPath, err := SafeResolvePath(s.cfg.UploadDir, filepath.Join(subdir, storedFilename)) - if err != nil { - return nil, fmt.Errorf("invalid upload path: %w", err) - } + // Build the storage key (e.g., "images/20240101_uuid.jpg") + key := subdir + "/" + storedFilename // Read all file content into memory for potential encryption fileData, err := io.ReadAll(src) @@ -151,13 +149,13 @@ func (s *StorageService) Upload(file *multipart.FileHeader, category string) (*U } } - // Write file content to disk - if err := os.WriteFile(destPath, fileData, 0644); err != nil { + // Write to backend + if err := s.backend.Write(key, fileData); err != nil { return nil, fmt.Errorf("failed to save file: %w", err) } written := int64(len(fileData)) - // Generate URL (always uses the original filename without .enc suffix for the public URL) + // Generate URL (always uses the original filename without .enc suffix) url := fmt.Sprintf("%s/%s/%s", s.cfg.BaseURL, subdir, newFilename) log.Info(). @@ -165,6 +163,7 @@ func (s *StorageService) Upload(file *multipart.FileHeader, category string) (*U Str("category", category). Int64("size", written). Str("mime_type", mimeType). + Bool("s3", s.cfg.IsS3()). Msg("File uploaded successfully") return &UploadResult{ @@ -183,33 +182,24 @@ func (s *StorageService) ReadFile(storedURL string) ([]byte, string, error) { return nil, "", fmt.Errorf("empty file URL") } - // Strip base URL prefix to get relative path - relativePath := strings.TrimPrefix(storedURL, s.cfg.BaseURL) - relativePath = strings.TrimPrefix(relativePath, "/") + // Strip base URL prefix to get relative key + relativeKey := strings.TrimPrefix(storedURL, s.cfg.BaseURL) + relativeKey = strings.TrimPrefix(relativeKey, "/") // Try .enc variant first, then plain file - var fullPath string + var data []byte var encrypted bool + var err error - encPath, err := SafeResolvePath(s.cfg.UploadDir, relativePath+".enc") + data, err = s.backend.Read(relativeKey + ".enc") if err == nil { - if _, statErr := os.Stat(encPath); statErr == nil { - fullPath = encPath - encrypted = true - } - } - - if fullPath == "" { - plainPath, err := SafeResolvePath(s.cfg.UploadDir, relativePath) + encrypted = true + } else { + // Fall back to plain file + data, err = s.backend.Read(relativeKey) if err != nil { - return nil, "", fmt.Errorf("invalid file path: %w", err) + return nil, "", fmt.Errorf("failed to read file: %w", err) } - fullPath = plainPath - } - - data, err := os.ReadFile(fullPath) - if err != nil { - return nil, "", fmt.Errorf("failed to read file: %w", err) } // Decrypt if this is an encrypted file @@ -231,58 +221,45 @@ func (s *StorageService) ReadFile(storedURL string) ([]byte, string, error) { // Delete removes a file from storage, handling both plain and .enc variants func (s *StorageService) Delete(fileURL string) error { - // Convert URL to file path relativePath := strings.TrimPrefix(fileURL, s.cfg.BaseURL) relativePath = strings.TrimPrefix(relativePath, "/") - // S-18: Use SafeResolvePath to prevent path traversal - fullPath, err := SafeResolvePath(s.cfg.UploadDir, relativePath) - if err != nil { - return fmt.Errorf("invalid file path: %w", err) - } + // Delete both plain and .enc variants (ignore not-found errors) + plainErr := s.backend.Delete(relativePath) + encErr := s.backend.Delete(relativePath + ".enc") - // Try to delete the plain file - plainDeleted := false - if err := os.Remove(fullPath); err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("failed to delete file: %w", err) - } - } else { - plainDeleted = true - log.Info().Str("path", fullPath).Msg("File deleted") + // Only return an error if both failed for reasons other than not-found + if plainErr != nil { + log.Debug().Err(plainErr).Str("key", relativePath).Msg("Delete plain file") } - - // Also try to delete the .enc variant - encPath, err := SafeResolvePath(s.cfg.UploadDir, relativePath+".enc") - if err == nil { - if err := os.Remove(encPath); err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("failed to delete encrypted file: %w", err) - } - } else { - log.Info().Str("path", encPath).Msg("Encrypted file deleted") - return nil - } - } - - if !plainDeleted { - // Neither file existed — that's OK - return nil + if encErr != nil { + log.Debug().Err(encErr).Str("key", relativePath+".enc").Msg("Delete enc file") } return nil } +// GetUploadDir returns the upload directory path. +// For S3 backends, returns empty string. +func (s *StorageService) GetUploadDir() string { + if lb, ok := s.backend.(*LocalBackend); ok { + return lb.BaseDir() + } + return s.cfg.UploadDir +} + +// SetEncryptionService sets the encryption service for encrypting files at rest +func (s *StorageService) SetEncryptionService(svc *EncryptionService) { + s.encryptionSvc = svc +} + // isAllowedType checks if the MIME type is in the allowed list. -// P-12: Uses the pre-parsed allowedTypes map for O(1) lookups instead of -// splitting the config string on every call. func (s *StorageService) isAllowedType(mimeType string) bool { _, ok := s.allowedTypes[mimeType] return ok } // mimeTypesCompatible checks if the claimed and detected MIME types are compatible. -// Two MIME types are compatible if they share the same primary type (e.g., both "image/*"). func (s *StorageService) mimeTypesCompatible(claimed, detected string) bool { claimedParts := strings.SplitN(claimed, "/", 2) detectedParts := strings.SplitN(detected, "/", 2) @@ -307,25 +284,24 @@ func (s *StorageService) getExtensionFromMimeType(mimeType string) string { return "" } -// GetUploadDir returns the upload directory path -func (s *StorageService) GetUploadDir() string { - return s.cfg.UploadDir -} - -// SetEncryptionService sets the encryption service for encrypting files at rest -func (s *StorageService) SetEncryptionService(svc *EncryptionService) { - s.encryptionSvc = svc +// parseAllowedTypes splits a comma-separated MIME type string into a set. +func parseAllowedTypes(types string) map[string]struct{} { + allowed := make(map[string]struct{}) + for _, t := range strings.Split(types, ",") { + trimmed := strings.TrimSpace(t) + if trimmed != "" { + allowed[trimmed] = struct{}{} + } + } + return allowed } // NewStorageServiceForTest creates a StorageService without creating directories. // This is intended only for unit tests that need a StorageService with a known config. func NewStorageServiceForTest(cfg *config.StorageConfig) *StorageService { - allowedTypes := make(map[string]struct{}) - for _, t := range strings.Split(cfg.AllowedTypes, ",") { - trimmed := strings.TrimSpace(t) - if trimmed != "" { - allowedTypes[trimmed] = struct{}{} - } + return &StorageService{ + cfg: cfg, + backend: nil, // tests that need a backend must set it up + allowedTypes: parseAllowedTypes(cfg.AllowedTypes), } - return &StorageService{cfg: cfg, allowedTypes: allowedTypes} }