diff --git a/config/conf.json b/config/conf.json index 59041f1..dd45321 100644 --- a/config/conf.json +++ b/config/conf.json @@ -8,11 +8,13 @@ "sslmode": "disable" }, "valkey": { - "host": "127.0.0.1", - "port": "6379", - "max_conns": 50, - "timeout_ms": 5000, - "password": "the_valkey_password" - }, - "temp": "value" + "host": "127.0.0.1", + "port": "6379", + "max_conns": 50, + "timeout_ms": 5000, + "password": "the_valkey_password" + }, + "temp": "value", + "dataset_id": "", + "excluded_dataset_ids": "" } diff --git a/config/config.go b/config/config.go index 79d212e..404237e 100644 --- a/config/config.go +++ b/config/config.go @@ -21,9 +21,11 @@ type Config struct { Port string `json:"port"` MaxConns int `json:"max_conns"` TimeoutMs int `json:"timeout_ms"` - Password string `json:"password"` // Add this line + Password string `json:"password"` } `json:"valkey"` - Temp string `json:"temp"` + Temp string `json:"temp"` + DatasetId string `json:"dataset_id"` + ExcludedDatasetIds string `json:"excluded_dataset_ids"` } func LoadConfig(file string) (Config, error) { @@ -80,5 +82,13 @@ func LoadConfig(file string) (Config, error) { } } + // Override datasetId and excludedDatasetIds with environment variables + if datasetId := os.Getenv("DATASET_ID"); datasetId != "" { + config.DatasetId = datasetId + } + if excludedDatasetIds := os.Getenv("EXCLUDED_DATASET_IDS"); excludedDatasetIds != "" { + config.ExcludedDatasetIds = excludedDatasetIds + } + return config, nil } diff --git a/data/data.go b/data/data.go index b7ddee9..5727f6d 100644 --- a/data/data.go +++ b/data/data.go @@ -1,9 +1,10 @@ package data import ( - "log" "encoding/xml" + "log" "net/http" + "strings" ) type Data struct { @@ -126,13 +127,20 @@ type Data struct { } `xml:"ServiceDelivery"` } -func FetchData(timestamp string) (*Data, error) { +func FetchData(timestamp, datasetId, excludedDatasetIds string) (*Data, error) { client := &http.Client{} requestorId := "ti1-" + timestamp - url := "https://api.entur.io/realtime/v1/rest/et?useOriginalId=true&maxSize=100000&requestorId=" + requestorId - log.Println("Fetching data from URL:", url) - resp, err := client.Get(url) + baseURL := "https://api.entur.io/realtime/v1/rest/et?useOriginalId=true&maxSize=100000&requestorId=" + requestorId + + if datasetId != "" { + baseURL += "&datasetId=" + datasetId + } else if excludedDatasetIds != "" { + baseURL += "&excludedDatasetIds=" + strings.ReplaceAll(excludedDatasetIds, ",", "&excludedDatasetIds=") + } + + log.Println("Fetching data from URL:", baseURL) + resp, err := client.Get(baseURL) if err != nil { return nil, err } diff --git a/main.go b/main.go index a540855..b5c6515 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "log" + "ti1/config" "ti1/data" "ti1/database" "ti1/export" @@ -12,8 +13,14 @@ func main() { log.Println("ti1 v0.2.1") log.Println("Starting...") + // Load configuration + cfg, err := config.LoadConfig("config.json") + if err != nil { + log.Fatalf("Failed to load config: %v", err) + } + // Setup the database - err := database.SetupDB() + err = database.SetupDB() if err != nil { log.Fatalf("Database setup failed: %v", err) } @@ -25,7 +32,7 @@ func main() { for { start := time.Now() - data, err := data.FetchData(starttimestamp) + data, err := data.FetchData(starttimestamp, cfg.DatasetId, cfg.ExcludedDatasetIds) if err != nil { log.Fatal(err) }