didnt work well

config/db.go

@@ -4,7 +4,6 @@ import (
 	"database/sql"
 	"fmt"
 	"log"
-    "time"
 
 	_ "github.com/lib/pq"
 )
@@ -29,15 +28,6 @@ func ConnectToPostgreSQL() (*sql.DB, error) {
 
 	fmt.Println("Connection to PostgreSQL opened successfully :D")
 
-    // Set the maximum number of open connections to 20
-    db.SetMaxOpenConns(20)
-
-    // Set the maximum number of idle connections to 10
-    db.SetMaxIdleConns(10)
-
-    // Set the maximum connection lifetime to 1 hour
-    db.SetConnMaxLifetime(1 * time.Hour)
-
 	// Ping database to verify connection
 	err = db.Ping()
 

export/database.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"log"
 	"strings"
-    "sync"
 	"ti1/config"
 	"ti1/data"
 	"ti1/database"
@@ -30,40 +29,7 @@ func DBData(data *data.Data) {
 
 	// counters
 	var insertCount, updateCount, totalCount, estimatedCallInsertCount, estimatedCallUpdateCount, recordedCallInsertCount, recordedCallUpdateCount int
-    var mu sync.Mutex
 
-    // Create a worker pool
-    numWorkers := 10
-    jobs := make(chan []interface{}, numWorkers)
-    results := make(chan struct {
-        action string
-        id     int
-        err    error
-    }, numWorkers)
-
-    var wg sync.WaitGroup
-
-    // Worker function
-    worker := func() {
-        defer wg.Done()
-        for values := range jobs {
-            id, action, err := database.InsertOrUpdateEstimatedVehicleJourney(db, values)
-            results <- struct {
-                action string
-                id     int
-                err    error
-            }{action, id, err}
-        }
-    }
-
-    // Start workers
-    for i := 0; i < numWorkers; i++ {
-        wg.Add(1)
-        go worker()
-    }
-
-    // Send jobs to workers
-    go func() {
 	for _, journey := range data.ServiceDelivery.EstimatedTimetableDelivery[0].EstimatedJourneyVersionFrame.EstimatedVehicleJourney {
 		var values []interface{}
 		var datedVehicleJourneyRef, otherJson string
@@ -71,6 +37,7 @@ func DBData(data *data.Data) {
 		values = append(values, sid)
 		values = append(values, journey.RecordedAtTime)
 		values = append(values, journey.LineRef)
+		//had to add to lowercase cus some values vary in case and it was causing duplicates
 		values = append(values, strings.ToLower(journey.DirectionRef))
 		values = append(values, journey.DataSource)
 
@@ -91,7 +58,10 @@ func DBData(data *data.Data) {
 		values = append(values, journey.VehicleRef)
 		values = append(values, journey.Cancellation)
 
+		// Create a map to hold the JSON object for the current journey
 		jsonObject := make(map[string]interface{})
+
+		// Add relevant fields to the JSON object
 		if journey.OriginName != "" {
 			jsonObject["OriginName"] = journey.OriginName
 		}
@@ -162,6 +132,7 @@ func DBData(data *data.Data) {
 			jsonObject["Via"] = journey.Via.PlaceName
 		}
 
+		// Convert the JSON object to a JSON string
 		jsonString, err := json.Marshal(jsonObject)
 		if err != nil {
 			log.Fatal(err)
@@ -169,18 +140,37 @@ func DBData(data *data.Data) {
 		otherJson = string(jsonString)
 		values = append(values, otherJson)
 
-            // Insert or update the record and get the id
+		// Insert or update the record
 		id, action, err := database.InsertOrUpdateEstimatedVehicleJourney(db, values)
 		if err != nil {
 			fmt.Printf("Error inserting/updating estimated vehicle journey: %v\n", err)
-                continue
 		} else {
 			if 1 == 0 {
 				fmt.Printf("Action: %s, ID: %d\n", action, id)
 			}
+
+			if action == "insert" {
+				insertCount++
+			} else if action == "update" {
+				updateCount++
+			}
+			totalCount = insertCount + updateCount
+
+			//fmt.Printf("Inserts: %d, Updates: %d, Total: %d\n", insertCount, updateCount, totalCount)
+			if totalCount%1000 == 0 {
+				fmt.Printf(
+					"Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n",
+					insertCount,
+					updateCount,
+					totalCount,
+					estimatedCallInsertCount,
+					estimatedCallUpdateCount,
+					recordedCallInsertCount,
+					recordedCallUpdateCount,
+				)
+			}
 		}
 
-            // Add the missing code here
 		for _, estimatedCall := range journey.EstimatedCalls {
 			for _, call := range estimatedCall.EstimatedCall {
 				var estimatedValues []interface{}
@@ -204,7 +194,7 @@ func DBData(data *data.Data) {
 
 				//9 estimated_data (JSON)
 				estimatedJsonObject := make(map[string]interface{})
-                    // data already logged
+				// data allrady loged
 				if call.ExpectedDepartureTime != "" {
 					estimatedJsonObject["ExpectedDepartureTime"] = call.ExpectedDepartureTime
 				}
@@ -294,7 +284,7 @@ func DBData(data *data.Data) {
 
 				// Insert or update the record
 				stringValues := make([]string, len(estimatedValues))
-                    for i, v := range stringValues {
+				for i, v := range estimatedValues {
 					stringValues[i] = fmt.Sprintf("%v", v)
 				}
 				interfaceValues := make([]interface{}, len(stringValues))
@@ -369,7 +359,7 @@ func DBData(data *data.Data) {
 
 				// Insert or update the record
 				stringValues := make([]string, len(recordedValues))
-                    for i, v := range stringValues {
+				for i, v := range recordedValues {
 					stringValues[i] = fmt.Sprintf("%v", v)
 				}
 				interfaceValues := make([]interface{}, len(stringValues))
@@ -394,44 +384,8 @@ func DBData(data *data.Data) {
 				}
 			}
 		}
-        }
-        close(jobs)
-    }()
 
-    // Collect results
-    go func() {
-        for result := range results {
-            if result.err != nil {
-                fmt.Printf("Error inserting/updating estimated vehicle journey: %v\n", result.err)
-            } else {
-                mu.Lock()
-                if result.action == "insert" {
-                    insertCount++
-                } else if result.action == "update" {
-                    updateCount++
 	}
-                totalCount = insertCount + updateCount
-                mu.Unlock()
-
-                if totalCount%1000 == 0 {
-                    fmt.Printf(
-                        "Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n",
-                        insertCount,
-                        updateCount,
-                        totalCount,
-                        estimatedCallInsertCount,
-                        estimatedCallUpdateCount,
-                        recordedCallInsertCount,
-                        recordedCallUpdateCount,
-                    )
-                }
-            }
-        }
-    }()
-
-    wg.Wait()
-    close(results)
-
 	fmt.Printf(
 		"DONE: Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n",
 		insertCount,
@@ -442,8 +396,10 @@ func DBData(data *data.Data) {
 		recordedCallInsertCount,
 		recordedCallUpdateCount,
 	)
-
+	// Create map to hold JSON
 	serviceDeliveryJsonObject := make(map[string]interface{})
+
+	// Add fields to JSON
 	serviceDeliveryJsonObject["Inserts"] = insertCount
 	serviceDeliveryJsonObject["Updates"] = updateCount
 	serviceDeliveryJsonObject["EstimatedCallInserts"] = estimatedCallInsertCount
@@ -451,11 +407,13 @@ func DBData(data *data.Data) {
 	serviceDeliveryJsonObject["RecordedCallInserts"] = recordedCallInsertCount
 	serviceDeliveryJsonObject["RecordedCallUpdates"] = recordedCallUpdateCount
 
+	// Convert JSON object to JSON string
 	serviceDeliveryJsonString, err := json.Marshal(serviceDeliveryJsonObject)
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	// Update ServiceDelivery data in database
 	err = database.UpdateServiceDeliveryData(db, sid, string(serviceDeliveryJsonString))
 	if err != nil {
 		log.Fatal(err)