Try lmdb

core/config.go

@@ -10,9 +10,10 @@ import (
 )
 
 type Config struct {
+	Database   string
+	UnixSocket string
 	Service    []scanner.ServiceConfig
 	Matrix     MatrixConfig
-	UnixSocket string
 }
 
 type MatrixConfig struct {

core/database.go

@@ -1,3 +1,63 @@
 package core
 
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"git.exozy.me/exozyme/status/scanner"
+	"github.com/PowerDNS/lmdb-go/lmdb"
+)
+
 // todo
+
+type Row struct {
+	time time.Time
+	data scanner.ServiceStatus
+}
+
+type Database = struct {
+	env *lmdb.Env
+	dbi lmdb.DBI
+}
+
+// type Database interface {
+// 	QueryAfter(url string, cutoff time.Time) []Row
+// 	RememberNow(scanner.ServiceStatusBatch) error
+// }
+
+func OpenDatabase(path string) (*Database, error) {
+	if path == "" {
+		return nil, fmt.Errorf("database path is empty. did you set it in config file?")
+	}
+	err := os.MkdirAll(path, 0755)
+	if err != nil {
+		return nil, err
+	}
+	env, err := lmdb.NewEnv() // will auto close
+	if err != nil {
+		return nil, err
+	}
+	err = env.SetMaxDBs(1)
+	if err != nil {
+		return nil, err
+	}
+	err = env.Open(path, lmdb.FixedMap, 0644)
+	if err != nil {
+		return nil, err
+	}
+	txn, err := env.BeginTxn(nil, 0)
+	if err != nil {
+		return nil, err
+	}
+	dbi, err := txn.CreateDBI("default")
+	if err != nil {
+		return nil, err
+	}
+	return &Database{env, dbi}, nil
+	// status := dbm.Open(path, true, nil)
+	// if !status.IsOK() {
+	// 	return nil, status
+	// }
+	// return dbm, nil
+}

go.mod

@@ -3,6 +3,7 @@ module git.exozy.me/exozyme/status
 go 1.21.1
 
 require (
+	github.com/PowerDNS/lmdb-go v1.9.1
 	github.com/cbroglie/mustache v1.4.0
 	github.com/pelletier/go-toml/v2 v2.1.0
 	github.com/prometheus-community/pro-bing v0.3.0

go.sum

@@ -1,3 +1,5 @@
+github.com/PowerDNS/lmdb-go v1.9.1 h1:5alOCXD5rjiOi0B0YfyPb2uWr8oAXF6LFiaPZ/d7KVY=
+github.com/PowerDNS/lmdb-go v1.9.1/go.mod h1:5beHlX2aYqXfMBI0+BBX3LDhV3YGHU5JgoDsirAFPlA=
 github.com/cbroglie/mustache v1.4.0 h1:Azg0dVhxTml5me+7PsZ7WPrQq1Gkf3WApcHMjMprYoU=
 github.com/cbroglie/mustache v1.4.0/go.mod h1:SS1FTIghy0sjse4DUVGV1k/40B1qE1XkD9DtDsHo9iM=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -43,14 +45,20 @@ golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
+golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
 golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

main.go

@@ -57,7 +57,7 @@ func rootPage(w http.ResponseWriter, r *http.Request) {
 	// get service status'
 	timeout := time.Second
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	data := scanner.TestMultipleServices(ctx, config.Service)
+	data := scanner.TestServiceBatch(ctx, config.Service)
 	cancel()
 
 	// don't uncomment this in production. verbose
@@ -79,6 +79,7 @@ func removeFile(path string) {
 }
 
 var config *core.Config
+var dbm *core.Database
 
 func main() {
 	var err error
@@ -87,6 +88,11 @@ func main() {
 		log.Fatal(err)
 	}
 
+	dbm, err = core.OpenDatabase(config.Database)
+	if err != nil {
+		log.Fatal(err)
+	}
+
 	http.HandleFunc("/", rootPage)
 
 	// serve public folder from embedfs

proto/config.toml.in

@@ -1,4 +1,7 @@
 # unixSocket = "/srv/http/pages/status"
+# database = "/opt/status-config/db"
+database = "/tmp/exozyme-status-test-db"
+
 service = [
   { description = "exozy.me", url = "https://exozy.me" },
   { description = "drgns.space", url = "https://drgns.space" },

scanner/index.go

@@ -68,7 +68,9 @@ func TestService(service ServiceConfig, ctx context.Context) ServiceStatus {
 	}
 }
 
-func TestMultipleServices(ctx Context, services []ServiceConfig) []ServiceStatus {
+type ServiceStatusBatch []ServiceStatus
+
+func TestServiceBatch(ctx Context, services []ServiceConfig) ServiceStatusBatch {
 	var wg sync.WaitGroup
 
 	var data []ServiceStatus = make([]ServiceStatus, len(services))