i have no idea wtf im doin

export/database.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"strings"
+	"sync"
 	"ti1/config"
 	"ti1/data"
 	"ti1/database"
@@ -29,15 +30,47 @@ func DBData(data *data.Data) {
 
 	// counters
 	var insertCount, updateCount, totalCount, estimatedCallInsertCount, estimatedCallUpdateCount, recordedCallInsertCount, recordedCallUpdateCount int
+	var mu sync.Mutex
 
-	for _, journey := range data.ServiceDelivery.EstimatedTimetableDelivery[0].EstimatedJourneyVersionFrame.EstimatedVehicleJourney {
+	// Create a worker pool
+	numWorkers := 10
+	jobs := make(chan []interface{}, numWorkers)
+	results := make(chan struct {
+		action string
+		id     int
+		err    error
+	}, numWorkers)
+
+	var wg sync.WaitGroup
+
+	// Worker function
+	worker := func() {
+		defer wg.Done()
+		for values := range jobs {
+			id, action, err := database.InsertOrUpdateEstimatedVehicleJourney(db, values)
+			results <- struct {
+				action string
+				id     int
+				err    error
+			}{action, id, err}
+		}
+	}
+
+	// Start workers
+	for i := 0; i < numWorkers; i++ {
+		wg.Add(1)
+		go worker()
+	}
+
+	// Send jobs to workers
+	go func() {
+		for _, journey := range data.ServiceDelivery.EstimatedTimetableDelivery[0].EstimatedVehicleJourney {
 			var values []interface{}
 			var datedVehicleJourneyRef, otherJson string
 
 			values = append(values, sid)
 			values = append(values, journey.RecordedAtTime)
 			values = append(values, journey.LineRef)
-		//had to add to lowercase cus some values vary in case and it was causing duplicates
 			values = append(values, strings.ToLower(journey.DirectionRef))
 			values = append(values, journey.DataSource)
 
@@ -58,10 +91,7 @@ func DBData(data *data.Data) {
 			values = append(values, journey.VehicleRef)
 			values = append(values, journey.Cancellation)
 
-		// Create a map to hold the JSON object for the current journey
 			jsonObject := make(map[string]interface{})
-
-		// Add relevant fields to the JSON object
 			if journey.OriginName != "" {
 				jsonObject["OriginName"] = journey.OriginName
 			}
@@ -132,7 +162,6 @@ func DBData(data *data.Data) {
 				jsonObject["Via"] = journey.Via.PlaceName
 			}
 
-		// Convert the JSON object to a JSON string
 			jsonString, err := json.Marshal(jsonObject)
 			if err != nil {
 				log.Fatal(err)
@@ -140,37 +169,9 @@ func DBData(data *data.Data) {
 			otherJson = string(jsonString)
 			values = append(values, otherJson)
 
-		// Insert or update the record
+			jobs <- values
-		id, action, err := database.InsertOrUpdateEstimatedVehicleJourney(db, values)
-		if err != nil {
-			fmt.Printf("Error inserting/updating estimated vehicle journey: %v\n", err)
-		} else {
-			if 1 == 0 {
-				fmt.Printf("Action: %s, ID: %d\n", action, id)
-			}
-
-			if action == "insert" {
-				insertCount++
-			} else if action == "update" {
-				updateCount++
-			}
-			totalCount = insertCount + updateCount
-
-			//fmt.Printf("Inserts: %d, Updates: %d, Total: %d\n", insertCount, updateCount, totalCount)
-			if totalCount%1000 == 0 {
-				fmt.Printf(
-					"Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n",
-					insertCount,
-					updateCount,
-					totalCount,
-					estimatedCallInsertCount,
-					estimatedCallUpdateCount,
-					recordedCallInsertCount,
-					recordedCallUpdateCount,
-				)
-			}
-		}
 
+			// Add the missing code here
 			for _, estimatedCall := range journey.EstimatedCalls {
 				for _, call := range estimatedCall.EstimatedCall {
 					var estimatedValues []interface{}
@@ -194,7 +195,7 @@ func DBData(data *data.Data) {
 
 					//9 estimated_data (JSON)
 					estimatedJsonObject := make(map[string]interface{})
-				// data allrady loged
+					// data already logged
 					if call.ExpectedDepartureTime != "" {
 						estimatedJsonObject["ExpectedDepartureTime"] = call.ExpectedDepartureTime
 					}
@@ -284,7 +285,7 @@ func DBData(data *data.Data) {
 
 					// Insert or update the record
 					stringValues := make([]string, len(estimatedValues))
-				for i, v := range estimatedValues {
+					for i, v := range stringValues {
 						stringValues[i] = fmt.Sprintf("%v", v)
 					}
 					interfaceValues := make([]interface{}, len(stringValues))
@@ -359,7 +360,7 @@ func DBData(data *data.Data) {
 
 					// Insert or update the record
 					stringValues := make([]string, len(recordedValues))
-				for i, v := range recordedValues {
+					for i, v := range stringValues {
 						stringValues[i] = fmt.Sprintf("%v", v)
 					}
 					interfaceValues := make([]interface{}, len(stringValues))
@@ -384,8 +385,44 @@ func DBData(data *data.Data) {
 					}
 				}
 			}
-
 		}
+		close(jobs)
+	}()
+
+	// Collect results
+	go func() {
+		for result := range results {
+			if result.err != nil {
+				fmt.Printf("Error inserting/updating estimated vehicle journey: %v\n", result.err)
+			} else {
+				mu.Lock()
+				if result.action == "insert" {
+					insertCount++
+				} else if result.action == "update" {
+					updateCount++
+				}
+				totalCount = insertCount + updateCount
+				mu.Unlock()
+
+				if totalCount%1000 == 0 {
+					fmt.Printf(
+						"Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n",
+						insertCount,
+						updateCount,
+						totalCount,
+						estimatedCallInsertCount,
+						estimatedCallUpdateCount,
+						recordedCallInsertCount,
+						recordedCallUpdateCount,
+					)
+				}
+			}
+		}
+	}()
+
+	wg.Wait()
+	close(results)
+
 	fmt.Printf(
 		"DONE: Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n",
 		insertCount,
@@ -396,10 +433,8 @@ func DBData(data *data.Data) {
 		recordedCallInsertCount,
 		recordedCallUpdateCount,
 	)
-	// Create map to hold JSON
-	serviceDeliveryJsonObject := make(map[string]interface{})
 
-	// Add fields to JSON
+	serviceDeliveryJsonObject := make(map[string]interface{})
 	serviceDeliveryJsonObject["Inserts"] = insertCount
 	serviceDeliveryJsonObject["Updates"] = updateCount
 	serviceDeliveryJsonObject["EstimatedCallInserts"] = estimatedCallInsertCount
@@ -407,13 +442,11 @@ func DBData(data *data.Data) {
 	serviceDeliveryJsonObject["RecordedCallInserts"] = recordedCallInsertCount
 	serviceDeliveryJsonObject["RecordedCallUpdates"] = recordedCallUpdateCount
 
-	// Convert JSON object to JSON string
 	serviceDeliveryJsonString, err := json.Marshal(serviceDeliveryJsonObject)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	// Update ServiceDelivery data in database
 	err = database.UpdateServiceDeliveryData(db, sid, string(serviceDeliveryJsonString))
 	if err != nil {
 		log.Fatal(err)