Purge old function execution history

config/config.go

@@ -10,6 +10,7 @@ const (
 	defaultPostgresMaxIdleConns           = 5
 	defaultPostgresConnMaxLifetimeSeconds = 1800
 	defaultPostgresConnMaxIdleTimeSeconds = 300
+	defaultRetentionDays                  = 7
 )
 
 var Current AppConfig
@@ -123,6 +124,8 @@ type AppConfig struct {
 	ActivateFlag string
 	// PluginsPath is the full qualified path where plugins are stored
 	PluginsPath string
+	// FunctionHistoryRetentionDays is the number of days to keep function execution history before cleanup.
+	FunctionHistoryRetentionDays int
 }
 
 func LoadConfig() AppConfig {
@@ -143,40 +146,41 @@ func LoadConfig() AppConfig {
 		PostgresConnMaxIdleTimeSeconds: envInt(
 			"POSTGRES_CONN_MAX_IDLE_TIME_SECONDS", defaultPostgresConnMaxIdleTimeSeconds,
 		),
-		MailProvider:             os.Getenv("MAIL_PROVIDER"),
-		MailpitSMTPAddr:          os.Getenv("MAILPIT_SMTP_ADDR"),
-		MailpitAPIURL:            os.Getenv("MAILPIT_API_URL"),
-		FromEmail:                os.Getenv("FROM_EMAIL"),
-		FromName:                 os.Getenv("FROM_NAME"),
-		StorageProvider:          os.Getenv("STORAGE_PROVIDER"),
-		LocalStorageURL:          os.Getenv("LOCAL_STORAGE_URL"),
-		RedisURL:                 os.Getenv("REDIS_URL"),
-		RedisHost:                os.Getenv("REDIS_HOST"),
-		RedisPassword:            os.Getenv("REDIS_PASSWORD"),
-		StripeKey:                os.Getenv("STRIPE_KEY"),
-		StripePriceIDIdea:        os.Getenv("STRIPE_PRICEID_IDEA"),
-		StripePriceIDLaunch:      os.Getenv("STRIPE_PRICEID_LAUNCH"),
-		StripePriceIDTraction:    os.Getenv("STRIPE_PRICEID_TRACTION"),
-		StripePriceIDGrowth:      os.Getenv("STRIPE_PRICEID_GROWTH"),
-		StripeWebhookSecret:      os.Getenv("STRIPE_WEBHOOK_SECRET"),
-		StripeRedirectFromPortal: os.Getenv("STRIPE_REDIRECT"),
-		TwilioAccountID:          os.Getenv("TWILIO_ACCOUNTSID"),
-		TwilioAuthToken:          os.Getenv("TWILIO_AUTHTOKEN"),
-		TwilioTestCellNumber:     os.Getenv("MY_CELL"),
-		TwilioNumber:             os.Getenv("TWILIO_NUMBER"),
-		S3AccessKey:              os.Getenv("S3_ACCESSKEY"),
-		S3SecretKey:              os.Getenv("S3_SECRETKEY"),
-		S3Endpoint:               os.Getenv("S3_ENDPOINT"),
-		S3Region:                 os.Getenv("S3_REGION"),
-		S3Bucket:                 os.Getenv("S3_BUCKET"),
-		S3CDNURL:                 os.Getenv("S3_CDN_URL"),
-		KeepPermissionInName:     os.Getenv("KEEP_PERM_COL_NAME") == "",
-		RoleAwareRowPermissions:  os.Getenv("ROLE_AWARE_ROW_PERMISSIONS") == "true",
-		LogConsoleLevel:          os.Getenv("LOG_CONSOLE_LEVEL"),
-		LogFilename:              os.Getenv("LOG_FILENAME"),
-		FullTextIndexFile:        os.Getenv("FTS_INDEX_FILE"),
-		ActivateFlag:             os.Getenv("ACTIVATE_FLAG"),
-		PluginsPath:              os.Getenv("PLUGINS_PATH"),
+		MailProvider:                 os.Getenv("MAIL_PROVIDER"),
+		MailpitSMTPAddr:              os.Getenv("MAILPIT_SMTP_ADDR"),
+		MailpitAPIURL:                os.Getenv("MAILPIT_API_URL"),
+		FromEmail:                    os.Getenv("FROM_EMAIL"),
+		FromName:                     os.Getenv("FROM_NAME"),
+		StorageProvider:              os.Getenv("STORAGE_PROVIDER"),
+		LocalStorageURL:              os.Getenv("LOCAL_STORAGE_URL"),
+		RedisURL:                     os.Getenv("REDIS_URL"),
+		RedisHost:                    os.Getenv("REDIS_HOST"),
+		RedisPassword:                os.Getenv("REDIS_PASSWORD"),
+		StripeKey:                    os.Getenv("STRIPE_KEY"),
+		StripePriceIDIdea:            os.Getenv("STRIPE_PRICEID_IDEA"),
+		StripePriceIDLaunch:          os.Getenv("STRIPE_PRICEID_LAUNCH"),
+		StripePriceIDTraction:        os.Getenv("STRIPE_PRICEID_TRACTION"),
+		StripePriceIDGrowth:          os.Getenv("STRIPE_PRICEID_GROWTH"),
+		StripeWebhookSecret:          os.Getenv("STRIPE_WEBHOOK_SECRET"),
+		StripeRedirectFromPortal:     os.Getenv("STRIPE_REDIRECT"),
+		TwilioAccountID:              os.Getenv("TWILIO_ACCOUNTSID"),
+		TwilioAuthToken:              os.Getenv("TWILIO_AUTHTOKEN"),
+		TwilioTestCellNumber:         os.Getenv("MY_CELL"),
+		TwilioNumber:                 os.Getenv("TWILIO_NUMBER"),
+		S3AccessKey:                  os.Getenv("S3_ACCESSKEY"),
+		S3SecretKey:                  os.Getenv("S3_SECRETKEY"),
+		S3Endpoint:                   os.Getenv("S3_ENDPOINT"),
+		S3Region:                     os.Getenv("S3_REGION"),
+		S3Bucket:                     os.Getenv("S3_BUCKET"),
+		S3CDNURL:                     os.Getenv("S3_CDN_URL"),
+		KeepPermissionInName:         os.Getenv("KEEP_PERM_COL_NAME") == "",
+		RoleAwareRowPermissions:      os.Getenv("ROLE_AWARE_ROW_PERMISSIONS") == "true",
+		LogConsoleLevel:              os.Getenv("LOG_CONSOLE_LEVEL"),
+		LogFilename:                  os.Getenv("LOG_FILENAME"),
+		FullTextIndexFile:            os.Getenv("FTS_INDEX_FILE"),
+		ActivateFlag:                 os.Getenv("ACTIVATE_FLAG"),
+		PluginsPath:                  os.Getenv("PLUGINS_PATH"),
+		FunctionHistoryRetentionDays: envInt("FUNCTION_HISTORY_RETENTION_DAYS", defaultRetentionDays),
 	}
 }
 

database/memory/function.go

@@ -112,5 +112,13 @@ func (m *Memory) RanFunction(dbName, id string, rh model.ExecHistory) error {
 	exists.LastRun = rh.Completed
 	exists.History = append(exists.History, rh)
 
+	history := make([]model.ExecHistory, 0, len(exists.History))
+	for _, h := range exists.History {
+		if h.Completed.IsZero() || !h.Completed.Before(model.FunctionHistoryRetentionCutoff()) {
+			history = append(history, h)
+		}
+	}
+	exists.History = history
+
 	return create(m, dbName, "sb_functions", id, exists)
 }

database/memory/function_test.go

@@ -262,3 +262,52 @@ func TestRanFunction(t *testing.T) {
 		t.Fatal("expected last run time to be set")
 	}
 }
+
+func TestRanFunctionRemoveOldHistory(t *testing.T) {
+	id, err := createFunction("test-run-retention-cleanup", "test-retention-cleanup")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	old := model.ExecHistory{
+		FunctionID: id,
+		Version:    1,
+		Started:    time.Now().AddDate(0, 0, -8),
+		Completed:  time.Now().AddDate(0, 0, -8),
+		Success:    true,
+		Output:     []string{"old"},
+	}
+	if err := datastore.RanFunction(confDBName, id, old); err != nil {
+		t.Fatal(err)
+	}
+
+	recent := model.ExecHistory{
+		FunctionID: id,
+		Version:    2,
+		Started:    time.Now().Add(-2 * time.Second),
+		Completed:  time.Now(),
+		Success:    true,
+		Output:     []string{"recent"},
+	}
+
+	if err := datastore.RanFunction(confDBName, id, recent); err != nil {
+		t.Fatal(err)
+	}
+
+	fn, err := datastore.GetFunctionByID(confDBName, id)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(fn.History) != 1 {
+		t.Fatalf("expected history to have 1 item, got %d", len(fn.History))
+	}
+
+	if fn.History[0].Version != 2 {
+		t.Fatalf("expected recent history to remain, got version %d", fn.History[0].Version)
+	}
+
+	if fn.LastRun.IsZero() {
+		t.Fatal("expected last run time to be set")
+	}
+}

database/mongo/function.go

@@ -287,5 +287,18 @@ func (mg *Mongo) RanFunction(dbName, id string, rh model.ExecHistory) error {
 	if err := res.Err(); err != nil {
 		return err
 	}
+
+	update = bson.M{
+		"$pull": bson.M{
+			"h": bson.M{
+				"c": bson.M{"$lt": model.FunctionHistoryRetentionCutoff()},
+			},
+		},
+	}
+
+	res = db.Collection("sb_functions").FindOneAndUpdate(mg.Ctx, filter, update)
+	if err := res.Err(); err != nil {
+		return err
+	}
 	return nil
 }

database/mongo/function_test.go

@@ -204,3 +204,52 @@ func TestRanFunction(t *testing.T) {
 		t.Fatal("expected last run time to be set")
 	}
 }
+
+func TestRanFunctionRemoveOldHistory(t *testing.T) {
+	id, err := createFunction("test-run-retention-cleanup", "test-retention-cleanup")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	old := model.ExecHistory{
+		FunctionID: id,
+		Version:    1,
+		Started:    time.Now().AddDate(0, 0, -8),
+		Completed:  time.Now().AddDate(0, 0, -8),
+		Success:    true,
+		Output:     []string{"old"},
+	}
+	if err := datastore.RanFunction(confDBName, id, old); err != nil {
+		t.Fatal(err)
+	}
+
+	recent := model.ExecHistory{
+		FunctionID: id,
+		Version:    2,
+		Started:    time.Now().Add(-2 * time.Second),
+		Completed:  time.Now(),
+		Success:    true,
+		Output:     []string{"recent"},
+	}
+
+	if err := datastore.RanFunction(confDBName, id, recent); err != nil {
+		t.Fatal(err)
+	}
+
+	fn, err := datastore.GetFunctionByID(confDBName, id)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(fn.History) != 1 {
+		t.Fatalf("expected history to have 1 item, got %d", len(fn.History))
+	}
+
+	if fn.History[0].Version != 2 {
+		t.Fatalf("expected recent history to remain, got version %d", fn.History[0].Version)
+	}
+
+	if fn.LastRun.IsZero() {
+		t.Fatal("expected last run time to be set")
+	}
+}

database/postgresql/function.go

@@ -244,6 +244,17 @@ func (pg *PostgreSQL) RanFunction(dbName, id string, rh model.ExecHistory) error
 		pq.Array(rh.Output),
 	)
 
+	if err != nil {
+		return err
+	}
+
+	qry = fmt.Sprintf(`
+		DELETE FROM %s.sb_function_logs
+		WHERE function_id = $1 AND completed < $2
+		`, dbName)
+
+	_, err = pg.DB.Exec(qry, id, model.FunctionHistoryRetentionCutoff())
+
 	return err
 }
 

database/postgresql/function_test.go

@@ -204,3 +204,52 @@ func TestRanFunction(t *testing.T) {
 		t.Fatal("expected last run time to be set")
 	}
 }
+
+func TestRanFunctionRemoveOldHistory(t *testing.T) {
+	id, err := createFunction("test-run-retention-cleanup", "test-retention-cleanup")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	old := model.ExecHistory{
+		FunctionID: id,
+		Version:    1,
+		Started:    time.Now().AddDate(0, 0, -8),
+		Completed:  time.Now().AddDate(0, 0, -8),
+		Success:    true,
+		Output:     []string{"old"},
+	}
+	if err := datastore.RanFunction(confDBName, id, old); err != nil {
+		t.Fatal(err)
+	}
+
+	recent := model.ExecHistory{
+		FunctionID: id,
+		Version:    2,
+		Started:    time.Now().Add(-2 * time.Second),
+		Completed:  time.Now(),
+		Success:    true,
+		Output:     []string{"recent"},
+	}
+
+	if err := datastore.RanFunction(confDBName, id, recent); err != nil {
+		t.Fatal(err)
+	}
+
+	fn, err := datastore.GetFunctionByID(confDBName, id)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(fn.History) != 1 {
+		t.Fatalf("expected history to have 1 item, got %d", len(fn.History))
+	}
+
+	if fn.History[0].Version != 2 {
+		t.Fatalf("expected recent history to remain, got version %d", fn.History[0].Version)
+	}
+
+	if fn.LastRun.IsZero() {
+		t.Fatal("expected last run time to be set")
+	}
+}

database/sqlite/function.go

@@ -248,6 +248,16 @@ func (sl *SQLite) RanFunction(dbName, id string, rh model.ExecHistory) error {
 		rh.Success,
 		pq.Array(rh.Output),
 	)
+	if err != nil {
+		return err
+	}
+
+	qry = fmt.Sprintf(`
+		DELETE FROM %s_sb_function_logs
+		WHERE function_id = $1 AND completed < $2
+		`, dbName)
+
+	_, err = sl.DB.Exec(qry, id, model.FunctionHistoryRetentionCutoff())
 
 	return err
 }

database/sqlite/function_test.go

@@ -204,3 +204,52 @@ func TestRanFunction(t *testing.T) {
 		t.Fatal("expected last run time to be set")
 	}
 }
+
+func TestRanFunctionRemoveOldHistory(t *testing.T) {
+	id, err := createFunction("test-run-retention-cleanup", "test-retention-cleanup")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	old := model.ExecHistory{
+		FunctionID: id,
+		Version:    1,
+		Started:    time.Now().AddDate(0, 0, -8),
+		Completed:  time.Now().AddDate(0, 0, -8),
+		Success:    true,
+		Output:     []string{"old"},
+	}
+	if err := datastore.RanFunction(confDBName, id, old); err != nil {
+		t.Fatal(err)
+	}
+
+	recent := model.ExecHistory{
+		FunctionID: id,
+		Version:    2,
+		Started:    time.Now().Add(-2 * time.Second),
+		Completed:  time.Now(),
+		Success:    true,
+		Output:     []string{"recent"},
+	}
+
+	if err := datastore.RanFunction(confDBName, id, recent); err != nil {
+		t.Fatal(err)
+	}
+
+	fn, err := datastore.GetFunctionByID(confDBName, id)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(fn.History) != 1 {
+		t.Fatalf("expected history to have 1 item, got %d", len(fn.History))
+	}
+
+	if fn.History[0].Version != 2 {
+		t.Fatalf("expected recent history to remain, got version %d", fn.History[0].Version)
+	}
+
+	if fn.LastRun.IsZero() {
+		t.Fatal("expected last run time to be set")
+	}
+}

function/runtime_search_test.go

@@ -423,7 +423,8 @@ type runtimeTestContext struct {
 func newRuntimeTestContext(t *testing.T, dbName, code string) runtimeTestContext {
 	t.Helper()
 
-	cfg := config.AppConfig{}
+	cfg := config.LoadConfig()
+	config.Current = cfg
 	logger.Setup(cfg)
 	pubsub := cache.NewDevCache()
 	datastore := memory.New(pubsub.PublishDocument).(*memory.Memory)

model/function.go

@@ -120,6 +120,10 @@ func (ex ExecData) GetSecrets() (map[string]string, error) {
 	return secrets, nil
 }
 
+func FunctionHistoryRetentionCutoff() time.Time {
+	return time.Now().AddDate(0, 0, -config.Current.FunctionHistoryRetentionDays)
+}
+
 func newFunctionSecretsCipher() (cipher.Block, error) {
 	key := []byte(config.Current.AppSecret)
 	switch len(key) {