From cf5ba5fefe2e5789649f1e7e0c7c0972945256a0 Mon Sep 17 00:00:00 2001
From: Peder Vatn Austad <peder.vatn.austad@gmail.com>
Date: Sat, 28 Dec 2024 19:29:21 +0100
Subject: [PATCH] normaliseing data to avoid duplicates

---
 data/data.go       |  2 +-
 export/database.go |  6 ++++--
 main.go            | 15 +++++++++------
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/data/data.go b/data/data.go
index bba0acc..f6ef84f 100644
--- a/data/data.go
+++ b/data/data.go
@@ -128,7 +128,7 @@ type Data struct {
 func FetchData() (*Data, error) {
 	client := &http.Client{}
 
-	resp, err := client.Get("https://api.entur.io/realtime/v1/rest/et?useOriginalId=true&maxSize=50000")
+	resp, err := client.Get("https://api.entur.io/realtime/v1/rest/et?useOriginalId=true&maxSize=100000")
 	if err != nil {
 		return nil, err
 	}
diff --git a/export/database.go b/export/database.go
index bbe482d..ec5fcd6 100644
--- a/export/database.go
+++ b/export/database.go
@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"strings"
 	"ti1/config"
 	"ti1/data"
 	"ti1/database"
@@ -36,7 +37,8 @@ func DBData(data *data.Data) {
 		values = append(values, sid)
 		values = append(values, journey.RecordedAtTime)
 		values = append(values, journey.LineRef)
-		values = append(values, journey.DirectionRef)
+		//had to add to lowercase cus some values vary in case and it was causing duplicates
+		values = append(values, strings.ToLower(journey.DirectionRef))
 		values = append(values, journey.DataSource)
 
 		if journey.FramedVehicleJourneyRef.DatedVehicleJourneyRef != "" {
@@ -155,7 +157,7 @@ func DBData(data *data.Data) {
 			totalCount = insertCount + updateCount
 
 			//fmt.Printf("Inserts: %d, Updates: %d, Total: %d\n", insertCount, updateCount, totalCount)
-			if totalCount%500 == 0 {
+			if totalCount%1000 == 0 {
 				fmt.Printf(
 					"Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n",
 					insertCount,
diff --git a/main.go b/main.go
index 50c7804..36bd9df 100644
--- a/main.go
+++ b/main.go
@@ -9,13 +9,16 @@ import (
 func main() {
 	//config.PrintDBConfig()
 
-	data, err := data.FetchData()
-	if err != nil {
-		log.Fatal(err)
-	}
+	for i := 0; i < 10; i++ {
+		data, err := data.FetchData()
+		if err != nil {
+			log.Fatal(err)
+		}
 
-	//export.ExportToCSV(data)
-	export.DBData(data)
+		//export.ExportToCSV(data)
+		export.DBData(data)
+	}
+	println(":)")
 	//export.PrintData(data)
 
 	//log.Printf("Data fetched successfully: %+v", data)