i have no idea wtf im doin
This commit is contained in:
pigwin
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"ti1/config"
|
||||
"ti1/data"
|
||||
"ti1/database"
|
||||
@@ -29,15 +30,47 @@ func DBData(data *data.Data) {
|
||||
|
||||
// counters
|
||||
var insertCount, updateCount, totalCount, estimatedCallInsertCount, estimatedCallUpdateCount, recordedCallInsertCount, recordedCallUpdateCount int
|
||||
var mu sync.Mutex
|
||||
|
||||
for _, journey := range data.ServiceDelivery.EstimatedTimetableDelivery[0].EstimatedJourneyVersionFrame.EstimatedVehicleJourney {
|
||||
// Create a worker pool
|
||||
numWorkers := 10
|
||||
jobs := make(chan []interface{}, numWorkers)
|
||||
results := make(chan struct {
|
||||
action string
|
||||
id int
|
||||
err error
|
||||
}, numWorkers)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Worker function
|
||||
worker := func() {
|
||||
defer wg.Done()
|
||||
for values := range jobs {
|
||||
id, action, err := database.InsertOrUpdateEstimatedVehicleJourney(db, values)
|
||||
results <- struct {
|
||||
action string
|
||||
id int
|
||||
err error
|
||||
}{action, id, err}
|
||||
}
|
||||
}
|
||||
|
||||
// Start workers
|
||||
for i := 0; i < numWorkers; i++ {
|
||||
wg.Add(1)
|
||||
go worker()
|
||||
}
|
||||
|
||||
// Send jobs to workers
|
||||
go func() {
|
||||
for _, journey := range data.ServiceDelivery.EstimatedTimetableDelivery[0].EstimatedVehicleJourney {
|
||||
var values []interface{}
|
||||
var datedVehicleJourneyRef, otherJson string
|
||||
|
||||
values = append(values, sid)
|
||||
values = append(values, journey.RecordedAtTime)
|
||||
values = append(values, journey.LineRef)
|
||||
//had to add to lowercase cus some values vary in case and it was causing duplicates
|
||||
values = append(values, strings.ToLower(journey.DirectionRef))
|
||||
values = append(values, journey.DataSource)
|
||||
|
||||
@@ -58,10 +91,7 @@ func DBData(data *data.Data) {
|
||||
values = append(values, journey.VehicleRef)
|
||||
values = append(values, journey.Cancellation)
|
||||
|
||||
// Create a map to hold the JSON object for the current journey
|
||||
jsonObject := make(map[string]interface{})
|
||||
|
||||
// Add relevant fields to the JSON object
|
||||
if journey.OriginName != "" {
|
||||
jsonObject["OriginName"] = journey.OriginName
|
||||
}
|
||||
@@ -132,7 +162,6 @@ func DBData(data *data.Data) {
|
||||
jsonObject["Via"] = journey.Via.PlaceName
|
||||
}
|
||||
|
||||
// Convert the JSON object to a JSON string
|
||||
jsonString, err := json.Marshal(jsonObject)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
@@ -140,37 +169,9 @@ func DBData(data *data.Data) {
|
||||
otherJson = string(jsonString)
|
||||
values = append(values, otherJson)
|
||||
|
||||
// Insert or update the record
|
||||
id, action, err := database.InsertOrUpdateEstimatedVehicleJourney(db, values)
|
||||
if err != nil {
|
||||
fmt.Printf("Error inserting/updating estimated vehicle journey: %v\n", err)
|
||||
} else {
|
||||
if 1 == 0 {
|
||||
fmt.Printf("Action: %s, ID: %d\n", action, id)
|
||||
}
|
||||
|
||||
if action == "insert" {
|
||||
insertCount++
|
||||
} else if action == "update" {
|
||||
updateCount++
|
||||
}
|
||||
totalCount = insertCount + updateCount
|
||||
|
||||
//fmt.Printf("Inserts: %d, Updates: %d, Total: %d\n", insertCount, updateCount, totalCount)
|
||||
if totalCount%1000 == 0 {
|
||||
fmt.Printf(
|
||||
"Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n",
|
||||
insertCount,
|
||||
updateCount,
|
||||
totalCount,
|
||||
estimatedCallInsertCount,
|
||||
estimatedCallUpdateCount,
|
||||
recordedCallInsertCount,
|
||||
recordedCallUpdateCount,
|
||||
)
|
||||
}
|
||||
}
|
||||
jobs <- values
|
||||
|
||||
// Add the missing code here
|
||||
for _, estimatedCall := range journey.EstimatedCalls {
|
||||
for _, call := range estimatedCall.EstimatedCall {
|
||||
var estimatedValues []interface{}
|
||||
@@ -194,7 +195,7 @@ func DBData(data *data.Data) {
|
||||
|
||||
//9 estimated_data (JSON)
|
||||
estimatedJsonObject := make(map[string]interface{})
|
||||
// data allrady loged
|
||||
// data already logged
|
||||
if call.ExpectedDepartureTime != "" {
|
||||
estimatedJsonObject["ExpectedDepartureTime"] = call.ExpectedDepartureTime
|
||||
}
|
||||
@@ -284,7 +285,7 @@ func DBData(data *data.Data) {
|
||||
|
||||
// Insert or update the record
|
||||
stringValues := make([]string, len(estimatedValues))
|
||||
for i, v := range estimatedValues {
|
||||
for i, v := range stringValues {
|
||||
stringValues[i] = fmt.Sprintf("%v", v)
|
||||
}
|
||||
interfaceValues := make([]interface{}, len(stringValues))
|
||||
@@ -359,7 +360,7 @@ func DBData(data *data.Data) {
|
||||
|
||||
// Insert or update the record
|
||||
stringValues := make([]string, len(recordedValues))
|
||||
for i, v := range recordedValues {
|
||||
for i, v := range stringValues {
|
||||
stringValues[i] = fmt.Sprintf("%v", v)
|
||||
}
|
||||
interfaceValues := make([]interface{}, len(stringValues))
|
||||
@@ -384,8 +385,44 @@ func DBData(data *data.Data) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
close(jobs)
|
||||
}()
|
||||
|
||||
// Collect results
|
||||
go func() {
|
||||
for result := range results {
|
||||
if result.err != nil {
|
||||
fmt.Printf("Error inserting/updating estimated vehicle journey: %v\n", result.err)
|
||||
} else {
|
||||
mu.Lock()
|
||||
if result.action == "insert" {
|
||||
insertCount++
|
||||
} else if result.action == "update" {
|
||||
updateCount++
|
||||
}
|
||||
totalCount = insertCount + updateCount
|
||||
mu.Unlock()
|
||||
|
||||
if totalCount%1000 == 0 {
|
||||
fmt.Printf(
|
||||
"Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n",
|
||||
insertCount,
|
||||
updateCount,
|
||||
totalCount,
|
||||
estimatedCallInsertCount,
|
||||
estimatedCallUpdateCount,
|
||||
recordedCallInsertCount,
|
||||
recordedCallUpdateCount,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
fmt.Printf(
|
||||
"DONE: Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n",
|
||||
insertCount,
|
||||
@@ -396,10 +433,8 @@ func DBData(data *data.Data) {
|
||||
recordedCallInsertCount,
|
||||
recordedCallUpdateCount,
|
||||
)
|
||||
// Create map to hold JSON
|
||||
serviceDeliveryJsonObject := make(map[string]interface{})
|
||||
|
||||
// Add fields to JSON
|
||||
serviceDeliveryJsonObject := make(map[string]interface{})
|
||||
serviceDeliveryJsonObject["Inserts"] = insertCount
|
||||
serviceDeliveryJsonObject["Updates"] = updateCount
|
||||
serviceDeliveryJsonObject["EstimatedCallInserts"] = estimatedCallInsertCount
|
||||
@@ -407,13 +442,11 @@ func DBData(data *data.Data) {
|
||||
serviceDeliveryJsonObject["RecordedCallInserts"] = recordedCallInsertCount
|
||||
serviceDeliveryJsonObject["RecordedCallUpdates"] = recordedCallUpdateCount
|
||||
|
||||
// Convert JSON object to JSON string
|
||||
serviceDeliveryJsonString, err := json.Marshal(serviceDeliveryJsonObject)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Update ServiceDelivery data in database
|
||||
err = database.UpdateServiceDeliveryData(db, sid, string(serviceDeliveryJsonString))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
|
||||
Reference in New Issue
Block a user