pigwin/ti1
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

normaliseing data to avoid duplicates

Browse Source
This commit is contained in:
Peder Vatn Austad
2024-12-28 19:29:21 +01:00
parent c44eae1938
commit cf5ba5fefe
3 changed files with 14 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
data/data.go
View File

@@ -128,7 +128,7 @@ type Data struct {
func FetchData() (*Data, error) { func FetchData() (*Data, error) {
client := &http.Client{} client := &http.Client{}
resp, err := client.Get("https://api.entur.io/realtime/v1/rest/et?useOriginalId=true&maxSize=50000") resp, err := client.Get("https://api.entur.io/realtime/v1/rest/et?useOriginalId=true&maxSize=100000")
if err != nil { if err != nil {
return nil, err return nil, err
} }

6
export/database.go
View File

@@ -4,6 +4,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"log" "log"
"strings"
"ti1/config" "ti1/config"
"ti1/data" "ti1/data"
"ti1/database" "ti1/database"
@@ -36,7 +37,8 @@ func DBData(data *data.Data) {
values = append(values, sid) values = append(values, sid)
values = append(values, journey.RecordedAtTime) values = append(values, journey.RecordedAtTime)
values = append(values, journey.LineRef) values = append(values, journey.LineRef)
values = append(values, journey.DirectionRef) //had to add to lowercase cus some values vary in case and it was causing duplicates
values = append(values, strings.ToLower(journey.DirectionRef))
values = append(values, journey.DataSource) values = append(values, journey.DataSource)
if journey.FramedVehicleJourneyRef.DatedVehicleJourneyRef != "" { if journey.FramedVehicleJourneyRef.DatedVehicleJourneyRef != "" {
@@ -155,7 +157,7 @@ func DBData(data *data.Data) {
totalCount = insertCount + updateCount totalCount = insertCount + updateCount
//fmt.Printf("Inserts: %d, Updates: %d, Total: %d\n", insertCount, updateCount, totalCount) //fmt.Printf("Inserts: %d, Updates: %d, Total: %d\n", insertCount, updateCount, totalCount)
if totalCount%500 == 0 { if totalCount%1000 == 0 {
fmt.Printf( fmt.Printf(
"Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n", "Inserts: %d, Updates: %d, Total: %d; estimatedCalls = I: %d U: %d; recordedCalls = I: %d U: %d\n",
insertCount, insertCount,

3
main.go
View File

@@ -9,6 +9,7 @@ import (
func main() { func main() {
//config.PrintDBConfig() //config.PrintDBConfig()
for i := 0; i < 10; i++ {
data, err := data.FetchData() data, err := data.FetchData()
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
@@ -16,6 +17,8 @@ func main() {
//export.ExportToCSV(data) //export.ExportToCSV(data)
export.DBData(data) export.DBData(data)
}
println(":)")
//export.PrintData(data) //export.PrintData(data)
//log.Printf("Data fetched successfully: %+v", data) //log.Printf("Data fetched successfully: %+v", data)