Add dataset ID and excluded dataset IDz

config/conf.json

@@ -8,11 +8,13 @@
         "sslmode": "disable"
     },
     "valkey": {
-		"host": "127.0.0.1",
+        "host": "127.0.0.1",
-		"port": "6379",
+        "port": "6379",
-		"max_conns": 50,
+        "max_conns": 50,
-		"timeout_ms": 5000,
+        "timeout_ms": 5000,
-        	"password": "the_valkey_password"
+        "password": "the_valkey_password"
-	},
+    },
-    "temp": "value"
+    "temp": "value",
+    "dataset_id": "",
+    "excluded_dataset_ids": ""
 }

config/config.go

@@ -21,9 +21,11 @@ type Config struct {
 		Port      string `json:"port"`
 		MaxConns  int    `json:"max_conns"`
 		TimeoutMs int    `json:"timeout_ms"`
-		Password  string `json:"password"` // Add this line
+		Password  string `json:"password"`
 	} `json:"valkey"`
-	Temp string `json:"temp"`
+	Temp             string `json:"temp"`
+	DatasetId        string `json:"dataset_id"`
+	ExcludedDatasetIds string `json:"excluded_dataset_ids"`
 }
 
 func LoadConfig(file string) (Config, error) {
@@ -80,5 +82,13 @@ func LoadConfig(file string) (Config, error) {
 		}
 	}
 
+	// Override datasetId and excludedDatasetIds with environment variables
+	if datasetId := os.Getenv("DATASET_ID"); datasetId != "" {
+		config.DatasetId = datasetId
+	}
+	if excludedDatasetIds := os.Getenv("EXCLUDED_DATASET_IDS"); excludedDatasetIds != "" {
+		config.ExcludedDatasetIds = excludedDatasetIds
+	}
+
 	return config, nil
 }

data/data.go

@@ -1,9 +1,10 @@
 package data
 
 import (
-	"log"
 	"encoding/xml"
+	"log"
 	"net/http"
+	"strings"
 )
 
 type Data struct {
@@ -126,13 +127,20 @@ type Data struct {
 	} `xml:"ServiceDelivery"`
 }
 
-func FetchData(timestamp string) (*Data, error) {
+func FetchData(timestamp, datasetId, excludedDatasetIds string) (*Data, error) {
 	client := &http.Client{}
 	requestorId := "ti1-" + timestamp
 
-	url := "https://api.entur.io/realtime/v1/rest/et?useOriginalId=true&maxSize=100000&requestorId=" + requestorId
+	baseURL := "https://api.entur.io/realtime/v1/rest/et?useOriginalId=true&maxSize=100000&requestorId=" + requestorId
-	log.Println("Fetching data from URL:", url)
+
-	resp, err := client.Get(url)
+	if datasetId != "" {
+		baseURL += "&datasetId=" + datasetId
+	} else if excludedDatasetIds != "" {
+		baseURL += "&excludedDatasetIds=" + strings.ReplaceAll(excludedDatasetIds, ",", "&excludedDatasetIds=")
+	}
+
+	log.Println("Fetching data from URL:", baseURL)
+	resp, err := client.Get(baseURL)
 	if err != nil {
 		return nil, err
 	}

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"log"
+	"ti1/config"
 	"ti1/data"
 	"ti1/database"
 	"ti1/export"
@@ -12,8 +13,14 @@ func main() {
 	log.Println("ti1 v0.2.1")
 	log.Println("Starting...")
 
+	// Load configuration
+	cfg, err := config.LoadConfig("config.json")
+	if err != nil {
+		log.Fatalf("Failed to load config: %v", err)
+	}
+
 	// Setup the database
-	err := database.SetupDB()
+	err = database.SetupDB()
 	if err != nil {
 		log.Fatalf("Database setup failed: %v", err)
 	}
@@ -25,7 +32,7 @@ func main() {
 	for {
 		start := time.Now()
 
-		data, err := data.FetchData(starttimestamp)
+		data, err := data.FetchData(starttimestamp, cfg.DatasetId, cfg.ExcludedDatasetIds)
 		if err != nil {
 			log.Fatal(err)
 		}