normaliseing data to avoid duplicates

data/data.go

@@ -128,7 +128,7 @@ type Data struct {
 func FetchData() (*Data, error) {
 	client := &http.Client{}
 
-	resp, err := client.Get("https://api.entur.io/realtime/v1/rest/et?useOriginalId=true&maxSize=50000")
+	resp, err := client.Get("https://api.entur.io/realtime/v1/rest/et?useOriginalId=true&maxSize=100000")
 	if err != nil {
 		return nil, err
 	}

export/database.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"strings"
 	"ti1/config"
 	"ti1/data"
 	"ti1/database"
@@ -36,7 +37,8 @@ func DBData(data *data.Data) {
 		values = append(values, sid)
 		values = append(values, journey.RecordedAtTime)
 		values = append(values, journey.LineRef)
-		values = append(values, journey.DirectionRef)
+		//had to add to lowercase cus some values vary in case and it was causing duplicates
+		values = append(values, strings.ToLower(journey.DirectionRef))
 		values = append(values, journey.DataSource)
 
 		if journey.FramedVehicleJourneyRef.DatedVehicleJourneyRef != "" {
@@ -155,7 +157,7 @@ func DBData(data *data.Data) {
 			totalCount = insertCount + updateCount
 
 			//fmt.Printf("Inserts: %d, Updates: %d, Total: %d\n", insertCount, updateCount, totalCount)
-			if totalCount%500 == 0 {
+			if totalCount%1000 == 0 {
 				fmt.Printf(
 					"Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n",
 					insertCount,

main.go

@@ -9,6 +9,7 @@ import (
 func main() {
 	//config.PrintDBConfig()
 
+	for i := 0; i < 10; i++ {
 		data, err := data.FetchData()
 		if err != nil {
 			log.Fatal(err)
@@ -16,6 +17,8 @@ func main() {
 
 		//export.ExportToCSV(data)
 		export.DBData(data)
+	}
+	println(":)")
 	//export.PrintData(data)
 
 	//log.Printf("Data fetched successfully: %+v", data)