Add dataset ID and excluded dataset IDz

config/conf.json

@@ -8,11 +8,13 @@
         "sslmode": "disable"
     },
     "valkey": {
-		"host": "127.0.0.1",
-		"port": "6379",
-		"max_conns": 50,
-		"timeout_ms": 5000,
-        	"password": "the_valkey_password"
-	},
-    "temp": "value"
+        "host": "127.0.0.1",
+        "port": "6379",
+        "max_conns": 50,
+        "timeout_ms": 5000,
+        "password": "the_valkey_password"
+    },
+    "temp": "value",
+    "dataset_id": "",
+    "excluded_dataset_ids": ""
 }

config/config.go

@@ -21,9 +21,11 @@ type Config struct {
 		Port      string `json:"port"`
 		MaxConns  int    `json:"max_conns"`
 		TimeoutMs int    `json:"timeout_ms"`
-		Password  string `json:"password"` // Add this line
+		Password  string `json:"password"`
 	} `json:"valkey"`
-	Temp string `json:"temp"`
+	Temp             string `json:"temp"`
+	DatasetId        string `json:"dataset_id"`
+	ExcludedDatasetIds string `json:"excluded_dataset_ids"`
 }
 
 func LoadConfig(file string) (Config, error) {
@@ -80,5 +82,13 @@ func LoadConfig(file string) (Config, error) {
 		}
 	}
 
+	// Override datasetId and excludedDatasetIds with environment variables
+	if datasetId := os.Getenv("DATASET_ID"); datasetId != "" {
+		config.DatasetId = datasetId
+	}
+	if excludedDatasetIds := os.Getenv("EXCLUDED_DATASET_IDS"); excludedDatasetIds != "" {
+		config.ExcludedDatasetIds = excludedDatasetIds
+	}
+
 	return config, nil
 }