Golang, convert embedded struct to array - arrays

Is there a way to convert struct to array of values in Golang?
for example if I have this kind of struct (not just this one):
type Model struct {
Id bson.ObjectId `bson:"_id,omitempty"`
CreatedAt time.Time `bson:",omitempty"`
UpdatedAt time.Time `bson:",omitempty"`
DeletedAt time.Time `bson:",omitempty"`
CreatedBy bson.ObjectId `bson:",omitempty"`
UpdatedBy bson.ObjectId `bson:",omitempty"`
DeletedBy bson.ObjectId `bson:",omitempty"`
Logs []bson.ObjectId `bson:",omitempty"`
}
type User struct {
Name string `bson:"name"`
Model `bson:",inline"`
}
The case was, I usually send the JSON to the browser with this format:
var iota = -1
var data = {
NAME: ++iota, ID: ++iota, CREATED_AT: ++iota, UPDATED_AT: ++iota, DELETED_AT: ++iota, // and so on
rows: [['kiz',1,'2014-01-01','2014-01-01','2014-01-01'],
['yui',2,'2014-01-01','2014-01-01','2014-01-01'],
['ham',3,'2014-01-01','2014-01-01','2014-01-01'] // and so on
]
};
Instead of:
var data = {
rows: [{NAME:'kiz',ID:1,CreatedAt:'2014-01-01',UpdatedAt:'2014-01-01',DeletedAt:'2014-01-01'},
{NAME:'yui',ID:2,CreatedAt:'2014-01-01',UpdatedAt:'2014-01-01',DeletedAt:'2014-01-01'},
{NAME:'ham',ID:3,CreatedAt:'2014-01-01',UpdatedAt:'2014-01-01',DeletedAt:'2014-01-01'} // and so on
]
}
Here's what I've tried:
import (
"github.com/kr/pretty"
//"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
"reflect"
"runtime"
"strings"
"time"
)
// copy the model from above
func Explain(variable interface{}) {
_, file, line, _ := runtime.Caller(1)
//res, _ := json.MarshalIndent(variable, " ", " ")
res := pretty.Formatter(variable)
fmt.Printf("%s:%d: %# v\n", file[len(FILE_PATH):], line, res)
//spew.Dump(variable)
}
func s2a(i interface{}) []interface{} { // taken from https://gist.github.com/tonyhb/5819315
iVal := reflect.ValueOf(i).Elem()
//typ := iVal.Type()
values := make([]interface{}, 0, iVal.NumField())
for i := 0; i < iVal.NumField(); i++ {
f := iVal.Field(i)
//tag := typ.Field(i).Tag.Get("tagname")
//fmt.Println(tag)
// name := typ.Field(i).Name
v := f.Interface()
switch v.(type) {
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, string, []byte, time.Time:
// do nothing
// case struct{}: // how to catch any embeeded struct?
case Model: // Model (or any embedded/nameless struct) should also converted to array
//arr := s2a() // invalid type assertion: f.(Model) (non-interface type reflect.Value on left)
//arr := s2a(f.Addr().(&Model)) // invalid type assertion: f.Addr().(&Model) (non-interface type reflect.Value on left)
// umm.. how to convert f back to Model?
//for _, e := range arr {
values = append(values, e)
//}
default: // struct? but also interface and map T_T
//v = s2a(&v)
}
values = append(values, v)
}
return values
}
func main() {
//sess, err := mgo.Dial("127.0.0.1")
//Check(err, "unable to connect")
//db := sess.DB("test")
//coll := db.C("coll1")
user := User{}
user.Id = bson.NewObjectId()
user.Name = "kis"
//changeInfo, err := coll.UpsertId(user.Id, user)
//Check(err, "failed to insert")
//Explain(changeInfo)
//Explain(s2a(changeInfo))
user.Name = "test"
Explain(user)
Explain(s2a(&user))
//err = coll.FindId(user.Id).One(&user)
//Check(err, "failed to fetch")
//Explain(user)
//Explain(s2a(&user))
user.CreatedAt = time.Now()
//err = coll.UpdateId(user.Id, user)
//Check(err, "failed to update")
Explain(changeInfo)
Explain(s2a(&user))
user.CreatedAt = user.DeletedAt
//err = coll.FindId(user.Id).One(&user)
//Check(err, "failed to fetch")
Explain(user)
Explain(s2a(&user))
}
Is there easy/fast way to convert struct to array (and if there struct embedded/inside it, converted to array also)?

If you are happy to specify a fixed order for the fields in the array representation, you could do this by implementing the json.Marshaler interface to customise its representation. For example:
func (u User) MarshalJSON() ([]byte, error) {
a := []interface{}{
u.Name,
u.Id,
...,
}
return json.Marshal(a)
}
Now when you marshal variables of this type, they will be represented as an array. If you want to also do the reverse (unmarshal an array into this struct), you will also need to implement the json.Unmarshaler interface. This could be done in a similar fashion, using json.Unmarshal to decode into a []interface{} slice and then pull out the values. Make sure UnmarshalJSON is declared to take a pointer receiver though, or your code won't work (you'll end up updating a copy of the struct rather than the struct itself).

Why not use reflect.Kind()? Here's the playground: http://play.golang.org/p/YjbsnB4eln

Use the reflect package.
Here's some playground code that'll work for one record (of any struct type), you can refactor it to work for a slice of records.
EDIT: (copy-pasted for good measure)
package main
import "fmt"
import "strings"
import "reflect"
type X struct {
Y string
Z int
}
func main() {
data := X{"yval",3}
expectedResult := `{"Y": 0, "Z": 1, "rows": [["yval", 3]]}`
fmt.Println(convert(data))
fmt.Println(expectedResult)
}
func convert(data interface{}) string {
v := reflect.ValueOf(data)
n := v.NumField()
st := reflect.TypeOf(data)
headers := make([]string, n)
for i := 0; i < n; i++ {
headers[i] = fmt.Sprintf(`"%s": %d`, st.Field(i).Name, i)
}
rowContents := make([]string, n)
for i := 0; i < n; i++ {
x := v.Field(i)
s := fmt.Sprintf("%v", x.Interface())
if x.Type().String() == "string" {
s = `"` + s + `"`
}
rowContents[i] = s
}
return "{" + strings.Join(headers, ", ") + `, "rows": [[` + strings.Join(rowContents, ", ") + "]]}"
}

Related

Decoding a JSON response that exists only out of an array

I have an problem with decoding an JSON respone. I've tried to solve this problem for a couple off weeks and can't find an working solution online.
This is my Go code that gets the response:
package main
import (
"fmt"
"time"
"strconv"
"encoding/json"
"net/http"
"io"
)
const (
binanceUrl_0 = "https://api.binance.com"
binanceUrl_1 = "https://api1.binance.com"
binanceUrl_2 = "https://api2.binance.com"
binanceUrl_3 = "https://api3.binance.com"
//select which url to use
binanceUrl = binanceUrl_0
binance_GetServerTime = binanceUrl + "/api/v3/time"
binance_Ping = binanceUrl + "/api/v3/ping"
binance_GetExhangeInfo = binanceUrl + "/api/v3/exchangeInfo"
binance_GetExhangeInfo_Symbol = binanceUrl + "/api/v3/exchangeInfo?symbol=BNBBTC"
binance_GetKlineData = binanceUrl + "/api/v1/klines"
)
type Binance_klines struct {
OpenTime int64
open float32
high float32
low float32
close float32
volume float32
CloseTime int64
QuoteVolume float32
NumTrades int64
TakerBaseVolume float32
TakerQuoteVolume float32
}
func GetKlines_wEndTime(symbol string, interval string, limit int, endTime time.Time) ([]Binance_klines, error) {
var url string
url = binance_GetKlineData + "?" +
"symbol=" + symbol +
"&interval=" + interval +
"&limit=" + strconv.FormatInt(int64(limit), 10) +
"&endTime=" + strconv.FormatInt(endTime.Unix(),10 ) + "000"
response, err := http.Get(url)
if err != nil {
return nil, err
}
data, err := respToKlines(response.Body)
if err != nil {
return nil, err
}
response.Body.Close()
return data, nil
}
func respToKlines(data io.Reader) ([]Binance_klines, error) {
var klines []Binance_klines
var decoded []interface{}
decoder := json.NewDecoder(data)
err := decoder.Decode(&decoded)
if err != nil {
return nil, err
}
//Attempt 1:
//kline = (decoded).([]Binance_klines)
//err: invalid operation: (decoded) (variable of type []interface{}) is not an interface
//Attempt 2:
for i:=0; i<len(decoded); i++ {
to_parse := decoded[i]
fmt.Println("to_parse",to_parse)
//prints: to_parse [1.66427838e+12 20982.91000000 20992.61000000 20977.90000000 20980.95000000 0.68063000 1.664278439999e+12 14282.75833530 57 0.27942000 5864.01792110 0]
var kline Binance_klines
kline = (to_parse).(Binance_klines)
//err: interface conversion: interface {} is []interface {}, not dsBinance.Binance_klines
fmt.Println("kline",kline)
}
return klines, nil
}
func main() {
result, err := GetKlines_wEndTime( "BTCEUR", "1m", 3, time.Now() )
fmt.Println(result, err)
}
This is an reponse I get (converted to string from bytes):
[[1664277480000,"20980.42000000","20984.06000000","20966.57000000","20970.14000000","6.10441000",1664277539999,"128041.93403330",142,"2.97844000","62486.29173860","0"],[1664277540000,"20969.14000000","20976.08000000","20955.69000000","20970.15000000","3.17365000",1664277599999,"66548.64583140",88,"2.39827000","50292.47196580","0"],[1664277600000,"20970.15000000","20970.15000000","20970.15000000","20970.15000000","0.00000000",1664277659999,"0.00000000",0,"0.00000000","0.00000000","0"]]
My question is, what am I doing wrong? When I use a tool like https://mholt.github.io/json-to-go/, it wants me to make an [][]interface{}. But in my for loop you can see that it prints an (in my eyes) a valid: []interface{} but i cannot convert it to an struct of type Binance_Klines.
Is something wrong with this line:
kline = (to_parse).(Binance_klines)
Or am I just misunderstanding something? What do I need to change to be able to use the type assertion? Or to just decode it at once to the right struct?
You cannot cast []interface{} to Binance_klines. So kline = (to_parse).(Binance_klines) fails. You have to write the translation yourself.
The data returned is a 2 dimensional array. Here is the json payload you have formatted. The types are a mix of string and float64, so Go uses interface{} to store the values.
[
[
1664277480000,
"20980.42000000",
"20984.06000000",
"20966.57000000",
"20970.14000000",
"6.10441000",
1664277539999,
"128041.93403330",
142,
"2.97844000",
"62486.29173860",
"0"
],
[
1664277540000,
"20969.14000000",
"20976.08000000",
"20955.69000000",
"20970.15000000",
"3.17365000",
1664277599999,
"66548.64583140",
88,
"2.39827000",
"50292.47196580",
"0"
],
[
1664277600000,
"20970.15000000",
"20970.15000000",
"20970.15000000",
"20970.15000000",
"0.00000000",
1664277659999,
"0.00000000",
0,
"0.00000000",
"0.00000000",
"0"
]
]
The json decoder cannot convert this into your Binance_klines struct. But you can override the default unmarshal behavior yourself.
First I made a type for the sometimes quoted numbers, sometimes not.
type BinanceNumber string
func (b *BinanceNumber) UnmarshalJSON(data []byte) error {
*b = BinanceNumber(strings.Trim(string(data), "\""))
return nil
}
func (b BinanceNumber) Float64() float64 {
f, err := strconv.ParseFloat(string(b), 64)
if err != nil {
panic(err)
}
return f
}
func (b BinanceNumber) Int64() int64 {
i, err := strconv.ParseInt(string(b), 10, 64)
if err != nil {
panic(err)
}
return i
}
Then you override the Binance_klines unmarshal.
func (b *Binance_klines) UnmarshalJSON(data []byte) error {
var array []BinanceNumber
err := json.Unmarshal(data, &array)
if err != nil {
return err
}
b.OpenTime = array[0].Int64()
b.Open = float32(array[1].Float64())
b.High = float32(array[2].Float64())
b.Low = float32(array[3].Float64())
b.Close = float32(array[4].Float64())
b.Volume = float32(array[5].Float64())
b.CloseTime = array[6].Int64()
b.QuoteVolume = float32(array[7].Float64())
b.NumTrades = array[8].Int64()
b.TakerBaseVolume = float32(array[9].Float64())
b.TakerQuoteVolume = float32(array[10].Float64())
return nil
}
Putting it all together: https://go.dev/play/p/SGGbWEUFxJr.
This part:
var kline Binance_klines
kline = (to_parse).(Binance_klines)
needs to become
var kline Binance_klines
kline.OpenTimer = to_parse[0].(int64)
kline.open = strconv.ParseFloat(to_parse[1].(string), 64)
...
You are not receiving the json representation of your Binance_klines struct but a slice of any (a mix or numbers and strings).

Index out of range trying to add to slice

var bar string
var i int
var a []string
for foo, _ := reader.NextWord(); foo != bar; foo, _ = reader.NextWord() {
bar = foo
fmt.Print(foo)
a[i] = foo
i++
}
Shouldn't this be creating a nil slice and then adding the value to the appropriate place? I keep getting index out of range so I assume it's not adding to a[i]...
Checking length first with
if len(a) > 0 {
a[i] = foo
}
seems to help, but not getting the results I expected. I'll keep playing around.
Update: I did end up using append... I meant to update this thread but thank you both.
package main
import (
"fmt"
"log"
"os"
"strings"
"github.com/steven-ferrer/gonsole"
)
func main() {
file, err := os.Open("test.txt")
if err != nil {
log.Println(err)
}
defer file.Close()
reader := gonsole.NewReader(file)
// cycle through
var bar string
var i int
var quality []string = make([]string, 0)
var tempName []string = make([]string, 0)
var name []string = make([]string, 0)
for foo, _ := reader.NextWord(); foo != bar; foo, _ = reader.NextWord() {
bar = foo
if strings.Contains(foo, "(normal)") {
quality = append(quality, "normal")
for state := 0; state < 1; foo, _ = reader.NextWord() {
if foo == "|" {
state = 1
}
tempName = append(tempName, foo)
}
nameString := strings.Join(tempName, "")
name = append(name, nameString)
} else if strings.Contains(foo, "(unique)") {
quality = append(quality, "unique")
for state := 0; state < 1; foo, _ = reader.NextWord() {
if foo == "|" {
state = 1
}
tempName = append(tempName, foo)
}
nameString := strings.Join(tempName, "")
name = append(name, nameString)
} else if strings.Contains(foo, "(set)") {
quality = append(quality, "set")
for state := 0; state < 1; foo, _ = reader.NextWord() {
if foo == "|" {
state = 1
}
tempName = append(tempName, foo)
}
nameString := strings.Join(tempName, "")
name = append(name, nameString)
}
if tempName != nil {
tempName = nil // clear tempName
}
i++
}
Your slice a needs to be allocated utilizing make.
var a []string = make([]string, n)
where n is the size of the slice.
Removing some of the context-specific parts of your code, you should be using the append method against a dynamic-length slice.
package main
import (
"fmt"
"strings"
)
func main() {
book := "Lorem ipsum dolor sit amet"
var words []string
for _, word := range strings.Split(book, " ") {
words = append(words, word)
}
fmt.Printf("%+v\n", words)
}
https://play.golang.org/p/LMejsrmIGb9
If you know the number of values up front, the same can be achieved for a fixed length slice by using words := make([]string, 5), but I doubt this is what you want in this case.
The reason your code is causing you errors is because your slice isn't initialized at any given length, so your indexes don't yet exist. Generally when working with a slice, append is the method you want.
Opposite to this, when working with existing slices (ie, rangeing an slice), the reason you're able to set the values using indexes is because the index has already been allocated.

Golang code running really slow than same code in PHP

New to Golang, Yesterday I've started to play with Golang and wrote some code which was actually written in PHP. I just wanted to see difference in performance.
I am doing the exact same thing in PHP response is exact same in http request but the Golang is performing really slow even after compiling it.
I am trying to understand what things that I am using in Golang I shouldn't be using and how can I improve performance in this piece of Code.
I know Iterating over map is slow but PHP using hash maps for implementing multidimentional arrays, well. I can gurantee the sql queries I used were exact same copy pasted from PHP, machines are same, and loop numbers are same in both codes.
package main
import (
"database/sql"
"encoding/json"
"fmt"
_ "github.com/go-sql-driver/mysql"
"net/http"
"reflect"
"strings"
)
func main() {
db, err := sql.Open("mysql", "***:****#tcp(****:3306)/****")
fmt.Println(reflect.TypeOf(db))
checkErr(err)
fmt.Println("Handle Request setup... OK")
http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
jsonData, err := getListings(db)
checkErr(err)
w.Write([]byte(jsonData))
})
fmt.Println("Starting Server....")
fmt.Println("Listening on port 8081")
http.ListenAndServe(":8081", nil)
}
func getListings(db *sql.DB) ([]byte, error) {
var userId string = "142"
normalListings := sqlToArray(db, `******`)
manualListings := sqlToArray(db, "******")
var groupIds []string
for key := range manualListings {
groupId := "142," + manualListings[key]["group_id"]
if !stringInSlice(groupId, groupIds) {
groupIds = append(groupIds, groupId)
}
}
var groupIdsString string
groupIdsString = strings.Join(groupIds, ", ")
manualGroups := sqlToArray(db, "*****")
for key := range manualListings {
for key2 := range manualGroups {
groupId := "142," + manualListings[key]["group_id"]
if groupId == manualGroups[key]["ticket_id"] {
entry := make(map[string]string)
entry["ticket_id"] = manualListings[key]["listing_id"]
entry["date_created"] = manualGroups[key2]["date_created"]
normalListings = append(normalListings, entry)
}
}
}
return json.Marshal(normalListings)
}
func stringInSlice(a string, list []string) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}
func sqlToArray(db *sql.DB, sqlString string) []map[string]string {
rows, err := db.Query(sqlString)
checkErr(err)
columns, err := rows.Columns()
count := len(columns)
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)
tableData := make([]map[string]string, 0)
for rows.Next() {
for i := 0; i < count; i++ {
valuePtrs[i] = &values[i]
}
rows.Scan(valuePtrs...)
entry := make(map[string]string)
for i, col := range columns {
val := values[i]
b, ok := val.([]byte)
if ok {
entry[col] = string(b)
} else {
entry[col] = string(b)
}
}
tableData = append(tableData, entry)
}
return tableData
}
func checkErr(err error) {
if err != nil {
panic(err)
}
}
Edits:
Changed the code to use statically typed structs instead of using maps and Identified the problematic piece of code
New code:
package main
import (
"database/sql"
"encoding/json"
"fmt"
_ "github.com/go-sql-driver/mysql"
"net/http"
"strings"
)
type listingsType struct {
TicketId string
DateCreated string
}
func main() {
db, err := sql.Open("mysql", "******")
checkErr(err)
fmt.Println("Handle Request setup... OK")
http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
jsonData, err := getListings(db)
checkErr(err)
w.Write([]byte(jsonData))
})
fmt.Println("Starting Server....")
fmt.Println("Listening on port 8081")
http.ListenAndServe(":8081", nil)
}
func getListings(db *sql.DB) ([]byte, error) {
var userId string = "142"
normalListings := sqlToArray(db, `*****`)
manualListings := sqlToArray(db, "*****")
var groupIds []string
for _, elem := range manualListings {
groupId := "142," + elem.DateCreated
if !stringInSlice(groupId, groupIds) {
groupIds = append(groupIds, groupId)
}
}
var groupIdsString string
groupIdsString = strings.Join(groupIds, ", ")
fmt.Println(groupIdsString)
manualGroups := sqlToArray(db, "******")
for _, manualList := range manualListings {
for _, manualGroup := range manualGroups {
groupId := "142," + manualList.DateCreated
if groupId == manualGroup.TicketId {
var entry listingsType
entry.TicketId = manualList.TicketId
entry.DateCreated = manualGroup.DateCreated
normalListings = append(normalListings, entry)
}
}
}
return json.Marshal(normalListings)
}
func stringInSlice(a string, list []string) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}
func sqlToArray(db *sql.DB, sqlString string) []listingsType {
rows, err := db.Query(sqlString)
checkErr(err)
tableData := []listingsType{}
for rows.Next() {
var entry listingsType
rows.Scan(&entry.TicketId, &entry.DateCreated)
tableData = append(tableData, entry)
}
return tableData
}
func checkErr(err error) {
if err != nil {
panic(err)
}
}
Problematic piece of code
As soon as I comment the following block of code the my code performs just fine.
Any idea what is wrong with this loop ?
for _, manualList := range manualListings {
for _, manualGroup := range manualGroups {
groupId := "142," + manualList.DateCreated
if groupId == manualGroup.TicketId {
var entry listingsType
entry.TicketId = manualList.TicketId
entry.DateCreated = manualGroup.DateCreated
normalListings = append(normalListings, entry)
}
}
}
Profiling Result
Ok So got it fixed by the way. I brought the request time from 5k+ MS to 500 MS, now finally my PHP code is slower which is 900 MS
I got rid of the inner loop to search by implementing a separate function to get SQL data in a different data structure in key value of maps instead of searching whole arrays I created the value as key which I was looking for in array, This way I got rid of the second loop which was making trouble by linear search on strings.
manualGroups := sqlToArraySpecial(db, "****")
for _, manualList := range manualListings {
//index := stringInSliceArray(manualList.DateCreated, manualGroups)
groupId := "142," + manualList.DateCreated
var entry listingsType
entry.TicketId = manualList.TicketId
entry.DateCreated = manualGroups[groupId]
normalListings = append(normalListings, entry)
}
and here is my new SQL function
func sqlToArraySpecial(db *sql.DB, sqlString string) map[string]string {
rows, err := db.Query(sqlString)
checkErr(err)
tableData := make(map[string]string)
for rows.Next() {
var date_created string
var ticket_id string
rows.Scan(&ticket_id, &date_created)
//fmt.Println(ticket_id)
tableData[ticket_id] = date_created
}
return tableData
}
although this is a dead post, I cannot help but note since no one else has (explicitly), and it's kinda' important to know why:
nested for loops exhibit quadratic running time complexity,
and as you stated, searching an array takes linear time,
so simply put:
the computation time will increase by the square of the total number of elements.
now to answer why this isn't the case in php -- well cause you were using a hash map which:
can be said, exhibits a constant time complexity
again, simply put this means that:
look up time is not related to the number of elements (aka the size of the collection).
see: big-o
with all that being said, please note:
i don't know php,
thus do not the know details regarding how the language implements arrays, and
im not an algorithm expert,
so please treat my post as a statement for the general case.
pce

Cannot pass []datastore.PropertyList to GetMulti function (datastore: src has invalid type)

I have wrote the following function, since passing map going to be dynamic I'm using datastore.PropertyList. Single insert works with PropertyList, but in Multiple insert an error is displayed as "datastore: src has invalid type"
Edited and added the full source
Where I have gone wrong?
package main
import (
"fmt"
"golang.org/x/net/context"
"golang.org/x/oauth2/google"
"google.golang.org/cloud"
"google.golang.org/cloud/datastore"
"io/ioutil"
)
func main() {
//Single Insert
// var map0 map[string]interface{}
// map0 = make(map[string]interface{})
// map0["Id"] = "600"
// map0["Name"] = "Prasad"
// map0["Age"] = 23
// setOneDataStore(map0)
//Multiple Insert
var allMaps []map[string]interface{}
allMaps = make([]map[string]interface{}, 2)
var map1 map[string]interface{}
map1 = make(map[string]interface{})
map1["Id"] = "700"
map1["Name"] = "Jay"
map1["Age"] = 23
var map2 map[string]interface{}
map2 = make(map[string]interface{})
map2["Id"] = "800"
map2["Name"] = "Peter"
map2["Age"] = 30
allMaps[0] = map1
allMaps[1] = map2
setManyDataStore(allMaps)
}
func getDataStoreClient() (client *datastore.Client, err error) {
keyFile := "JAYWORLD-30y4f7c347pq.json"
projectID := "jay-world"
jsonKey, err := ioutil.ReadFile(keyFile)
if err != nil {
fmt.Println(err.Error())
} else {
conf, err := google.JWTConfigFromJSON(
jsonKey,
datastore.ScopeDatastore,
datastore.ScopeUserEmail,
)
if err != nil {
fmt.Println(err.Error())
} else {
ctx := context.Background()
client, err = datastore.NewClient(ctx, projectID, cloud.WithTokenSource(conf.TokenSource(ctx)))
if err != nil {
fmt.Println(err.Error())
}
}
}
return
}
func setManyDataStore(Objects []map[string]interface{}) {
ctx := context.Background()
client, err := getDataStoreClient() //have connection code in another function
ctx = datastore.WithNamespace(ctx, "CompanyA")
if err == nil {
var keys []*datastore.Key
keys = make([]*datastore.Key, len(Objects))
var propArray []datastore.PropertyList
propArray = make([]datastore.PropertyList, len(Objects))
for index := 0; index < len(Objects); index++ {
keys[index] = datastore.NewKey(ctx, "users", Objects[index]["Id"].(string), 0, nil)
props := datastore.PropertyList{}
for key, value := range Objects[index] {
props = append(props, datastore.Property{Name: key, Value: value})
}
propArray[index] = props
}
if _, err := client.PutMulti(ctx, keys, propArray); err != nil {
fmt.Println(err.Error())
} else {
fmt.Println("Success!")
}
} else {
fmt.Println("Connection Failed")
}
}
func setOneDataStore(Object map[string]interface{}) {
ctx := context.Background()
client, err := getDataStoreClient() //have connection code in another function
ctx = datastore.WithNamespace(ctx, "CompanyA")
key := datastore.NewKey(ctx, "users", Object["Id"].(string), 0, nil)
props := datastore.PropertyList{}
for key, value := range Object {
props = append(props, datastore.Property{Name: key, Value: value})
}
_, err = client.Put(ctx, key, &props)
if err != nil {
fmt.Println(err.Error())
} else {
fmt.Println("Success!")
}
}
As the error indicates: "datastore: src has invalid type" the value you pass as src (&propArray) has invalid type.
You can't pass a value of type *[]datastore.PropertyList as the src parameter of Client.PutMulti(). Quoting from the doc:
src must satisfy the same conditions as the dst argument to GetMulti.
And the conditions of dst at Client.GetMulti():
dst must be a []S, []*S, []I or []P, for some struct type S, some interface type I, or some non-interface non-pointer type P such that P or *P implements PropertyLoadSaver. If an []I, each element must be a valid dst for Get: it must be a struct pointer or implement PropertyLoadSaver.
So you can't pass a pointer to slice, but you can and you should pass a slice (drop the address & operator):
if _, err := client.PutMulti(ctx, keys, propArray); err != nil {
fmt.Println(err.Error())
} else {
fmt.Println("Success!")
}
Note: Seemingly you are creating keys with specific ids (which is the index). But if both the key name and id is the zero value, that is considered to be an incomplete key. Your first key will be an incomplete key as your index starts with 0. You might want to use different ids than the index.

decoding nested json objects in go

I found some posts on how to decoding json nested objects in go, I tried to apply the answers to my problem, but I only managed to find a partial solution.
My json file look like this:
{
"user":{
"gender":"male",
"age":"21-30",
"id":"80b1ea88-19d7-24e8-52cc-65cf6fb9b380"
},
"trials":{
"0":{"index":0,"word":"WORD 1","Time":3000,"keyboard":true,"train":true,"type":"A"},
"1":{"index":1,"word":"WORD 2","Time":3000,"keyboard":true,"train":true,"type":"A"},
},
"answers":{
"training":[
{"ans":0,"RT":null,"gtAns":"WORD 1","correct":0},
{"ans":0,"RT":null,"gtAns":"WORD 2","correct":0}
],
"test":[
{"ans":0,"RT":null,"gtAns":true,"correct":0},
{"ans":0,"RT":null,"gtAns":true,"correct":0}
]
}
}
Basically I need to parse the information inside it and save them into go structure. With the code below I managed to extract the user information, but it looks too complicated to me and it won't be easy to apply the same thing to the "answers" fields which contains 2 arrays with more than 100 entries each. Here the code I'm using now:
type userDetails struct {
Id string `json:"id"`
Age string `json:"age"`
Gender string `json:"gender"`
}
type jsonRawData map[string]interface {
}
func getJsonContent(r *http.Request) ( userDetails) {
defer r.Body.Close()
jsonBody, err := ioutil.ReadAll(r.Body)
var userDataCurr userDetails
if err != nil {
log.Printf("Couldn't read request body: %s", err)
} else {
var f jsonRawData
err := json.Unmarshal(jsonBody, &f)
if err != nil {
log.Printf("Error unmashalling: %s", err)
} else {
user := f["user"].(map[string]interface{})
userDataCurr.Id = user["id"].(string)
userDataCurr.Gender = user["gender"].(string)
userDataCurr.Age = user["age"].(string)
}
}
return userDataCurr
}
Any suggestions? Thanks a lot!
You're doing it the hard way by using interface{} and not taking advantage of what encoding/json gives you.
I'd do it something like this (note I assumed there was an error with the type of the "gtAns" field and I made it a boolean, you don't give enough information to know what to do with the "RT" field):
package main
import (
"encoding/json"
"fmt"
"io"
"log"
"strconv"
"strings"
)
const input = `{
"user":{
"gender":"male",
"age":"21-30",
"id":"80b1ea88-19d7-24e8-52cc-65cf6fb9b380"
},
"trials":{
"0":{"index":0,"word":"WORD 1","Time":3000,"keyboard":true,"train":true,"type":"A"},
"1":{"index":1,"word":"WORD 2","Time":3000,"keyboard":true,"train":true,"type":"A"}
},
"answers":{
"training":[
{"ans":0,"RT":null,"gtAns":true,"correct":0},
{"ans":0,"RT":null,"gtAns":true,"correct":0}
],
"test":[
{"ans":0,"RT":null,"gtAns":true,"correct":0},
{"ans":0,"RT":null,"gtAns":true,"correct":0}
]
}
}`
type Whatever struct {
User struct {
Gender Gender `json:"gender"`
Age Range `json:"age"`
ID IDString `json:"id"`
} `json:"user"`
Trials map[string]struct {
Index int `json:"index"`
Word string `json:"word"`
Time int // should this be a time.Duration?
Train bool `json:"train"`
Type string `json:"type"`
} `json:"trials"`
Answers map[string][]struct {
Answer int `json:"ans"`
RT json.RawMessage // ??? what type is this
GotAnswer bool `json:"gtAns"`
Correct int `json:"correct"`
} `json:"answers"`
}
// Using some custom types to show custom marshalling:
type IDString string // TODO custom unmarshal and format/error checking
type Gender int
const (
Male Gender = iota
Female
)
func (g *Gender) UnmarshalJSON(b []byte) error {
var s string
err := json.Unmarshal(b, &s)
if err != nil {
return err
}
switch strings.ToLower(s) {
case "male":
*g = Male
case "female":
*g = Female
default:
return fmt.Errorf("invalid gender %q", s)
}
return nil
}
func (g Gender) MarshalJSON() ([]byte, error) {
switch g {
case Male:
return []byte(`"male"`), nil
case Female:
return []byte(`"female"`), nil
default:
return nil, fmt.Errorf("invalid gender %v", g)
}
}
type Range struct{ Min, Max int }
func (r *Range) UnmarshalJSON(b []byte) error {
// XXX could be improved
_, err := fmt.Sscanf(string(b), `"%d-%d"`, &r.Min, &r.Max)
return err
}
func (r Range) MarshalJSON() ([]byte, error) {
return []byte(fmt.Sprintf(`"%d-%d"`, r.Min, r.Max)), nil
// Or:
b := make([]byte, 0, 8)
b = append(b, '"')
b = strconv.AppendInt(b, int64(r.Min), 10)
b = append(b, '-')
b = strconv.AppendInt(b, int64(r.Max), 10)
b = append(b, '"')
return b, nil
}
func fromJSON(r io.Reader) (Whatever, error) {
var x Whatever
dec := json.NewDecoder(r)
err := dec.Decode(&x)
return x, err
}
func main() {
// Use http.Get or whatever to get an io.Reader,
// (e.g. response.Body).
// For playground, substitute a fixed string
r := strings.NewReader(input)
// If you actually had a string or []byte:
// var x Whatever
// err := json.Unmarshal([]byte(input), &x)
x, err := fromJSON(r)
if err != nil {
log.Fatal(err)
}
fmt.Println(x)
fmt.Printf("%+v\n", x)
b, err := json.MarshalIndent(x, "", " ")
if err != nil {
log.Fatal(err)
}
fmt.Printf("Re-marshalled: %s\n", b)
}
Playground
Of course if you want to reuse those sub-types you could pull them out of the "Whatever" type into their own named types.
Also, note the use of a json.Decoder rather than reading in all the data ahead of time. Usually try and avoid any use of ioutil.ReadAll unless you really need all the data at once.

Resources