What I'm trying to do:
Transform all arrays of length 1 in a JSON file to non arrays.
e.g.
Input: {"path": [{"secret/foo": [{"capabilities": ["read"]}]}]}
Output: {"path": {"secret/foo": {"capabilities": "read"}}}
I can't use Structs as the JSON format will vary...
Right now I've managed to at least detect the 1 length slices:
package main
import (
"encoding/json"
"fmt"
)
func findSingletons(value interface{}) {
switch value.(type) {
case []interface{}:
if len(value.([]interface{})) == 1 {
fmt.Println("1 length array found!", value)
}
for _, v := range value.([]interface{}) {
findSingletons(v)
}
case map[string]interface{}:
for _, v := range value.(map[string]interface{}) {
findSingletons(v)
}
}
}
func removeSingletonsFromJSON(input string) {
jsonFromInput := json.RawMessage(input)
jsonMap := make(map[string]interface{})
err := json.Unmarshal([]byte(jsonFromInput), &jsonMap)
if err != nil {
panic(err)
}
findSingletons(jsonMap)
fmt.Printf("JSON value of without singletons:%s\n", jsonMap)
}
func main() {
jsonParsed := []byte(`{"path": [{"secret/foo": [{"capabilities": ["read"]}]}]}`)
removeSingletonsFromJSON(string(jsonParsed))
fmt.Println(`Should have output {"path": {"secret/foo": {"capabilities": "read"}}}`)
}
Which outputs
1 length array found! [map[secret/foo:[map[capabilities:[read]]]]]
1 length array found! [map[capabilities:[read]]]
1 length array found! [read]
JSON value of without singletons:map[path:[map[secret/foo:[map[capabilities:[read]]]]]]
Should have output {"path": {"secret/foo": {"capabilities": "read"}}}
But I'm not sure how I can change them into non-arrays...
The type switch is your friend:
switch t := v.(type) {
case []interface{}:
if len(t) == 1 {
data[k] = t[0]
And you may use recursion to remove inside elements, like so:
func removeOneElementSlice(data map[string]interface{}) {
for k, v := range data {
switch t := v.(type) {
case []interface{}:
if len(t) == 1 {
data[k] = t[0]
if v, ok := data[k].(map[string]interface{}); ok {
removeOneElementSlice(v)
}
}
}
}
}
I would do this to convert
{"path":[{"secret/foo":[{"capabilities":["read"]}]}]}
to
{"path":{"secret/foo":{"capabilities":"read"}}}:
package main
import (
"encoding/json"
"fmt"
"log"
)
func main() {
s := `{"path":[{"secret/foo":[{"capabilities":["read"]}]}]}`
fmt.Println(s)
var data map[string]interface{}
if err := json.Unmarshal([]byte(s), &data); err != nil {
panic(err)
}
removeOneElementSlice(data)
buf, err := json.Marshal(data)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(buf)) //{"a":"a","n":7}
}
func removeOneElementSlice(data map[string]interface{}) {
for k, v := range data {
switch t := v.(type) {
case []interface{}:
if len(t) == 1 {
data[k] = t[0]
if v, ok := data[k].(map[string]interface{}); ok {
removeOneElementSlice(v)
}
}
}
}
}
I need to compute the hash (md5 is ok) for a large number of files. So, in Go I have this code:
package main
import (
"io"
"os"
"fmt"
"path/filepath"
"crypto/md5"
"encoding/hex"
)
func strSliceRemove(slice []string, str string) []string {
var tempSlice []string;
for _, item := range slice {
if item != str {
tempSlice = append(tempSlice, item)
}
}
return tempSlice
}
func fileMD5(path string) (string, error) {
var returnMD5String string
file, err := os.Open(path)
if err != nil {
return returnMD5String, err
}
defer file.Close()
hash := md5.New()
if _, err := io.Copy(hash, file); err != nil {
return returnMD5String, err
}
hashInBytes := hash.Sum(nil)[:16]
returnMD5String = hex.EncodeToString(hashInBytes)
return returnMD5String, nil
}
func main() {
var doRead func(string)
doRead = func(sd string) {
filepath.Walk(sd, func(path string, f os.FileInfo, err error) error {
resolvedPath, resolvedPathErr := filepath.EvalSymlinks(path)
if resolvedPathErr != nil {
return nil
}
if f.Mode()&os.ModeSymlink == os.ModeSymlink {
doRead(resolvedPath)
} else {
if !f.IsDir() {
md5, _ := fileMD5(path)
fmt.Printf("%s\n", md5)
}
}
return nil
})
}
doRead("/tmp/electron")
return
}
It hashes correctly 1400 files in almost one second. If I use my OSX md5 command line utility, it takes more than 10 times the time. It is 10 times slower:
for FILE in `find /tmp/electron`; do
if [ ! -d "$FILE" ]; then
md5 $FILE;
fi;
done;
I tried a basic c program that does the same (based on this answer How to calculate the MD5 hash of a large file in C?) and still the time seems more or less 10 seconds.
What kind of strategy / library does crypto/md5 use?
I'm trying to unmarshal a JSON array of the following type:
[
{"abc's": "n;05881364"},
{"abcoulomb": "n;13658345"},
{"abcs": "n;05881364"}
]
into a map[string]string. This question Golang parse JSON array into data structure almost answered my problem, but mine is a truly map, not an array of maps. Unmarshaling into a []map[string]string worked but I now get a map of map[string]string, not a simple map of string as it should be
There is no way to do it directly with the json package; you have to do the conversion yourself. This is simple:
package main
import (
"encoding/json"
"fmt"
)
func main() {
data := []byte(`
[
{"abc's": "n;05881364"},
{"abcoulomb": "n;13658345"},
{"abcs": "n;05881364"}
]
`)
var mapSlice []map[string]string
if err := json.Unmarshal(data, &mapSlice); err != nil {
panic(err)
}
resultingMap := map[string]string{}
for _, m := range mapSlice {
for k, v := range m {
resultingMap[k] = v
}
}
fmt.Println(resultingMap)
}
Output
map[abc's:n;05881364 abcoulomb:n;13658345 abcs:n;05881364]
An alternative (though very similar) to Alex's answer is to define your own type along with an UnmarshalJSON function.
package main
import (
"encoding/json"
"fmt"
)
type myMapping map[string]string
func (mm myMapping) UnmarshalJSON(b []byte) error {
var temp []map[string]string
if err := json.Unmarshal(b, &temp); err != nil {
return err
}
for _, m := range temp {
for k, v := range m {
mm[k] = v
}
}
return nil
}
func main() {
data := []byte(`
[
{"abc's": "n;05881364"},
{"abcoulomb": "n;13658345"},
{"abcs": "n;05881364"}
]`)
resultingMap := myMapping{}
if err := json.Unmarshal(data, &resultingMap); err != nil {
panic(err)
}
fmt.Println(resultingMap)
}
Playground
Fairly new at go.
I am trying to basically read all the files in directory and subdirectory, populate in a slice and then process further.
Here is the code (Main functions to look are Main and expandDirectory
package main
import (
"fmt"
"io/ioutil"
"os"
)
const dirname string = "MyDirectory"
func check(e error) {
if e != nil {
panic(e)
}
}
func expandDirectory(currentDirectory string, allFiles []os.FileInfo) []os.FileInfo {
files, e := ioutil.ReadDir(currentDirectory)
check(e)
for _, internalDir := range files {
switch mode := internalDir.Mode(); {
case mode.IsDir():
var filepath = currentDirectory + internalDir.Name() + "\\"
expandDirectory(filepath, allFiles)
case mode.IsRegular():
allFiles = append(allFiles, internalDir)
}
}
return allFiles
}
func main() {
allFiles := expandDirectory(dirname, make([]os.FileInfo, 5))
fmt.Printf("%v", cap(allFiles))
}
The final print will print 5 which is the initial value.
Just FYI, the expandDirectory function is just recursive function in case of directories and adding the file reference in case of a file.
Thanks for the help
Where am I going wrong
FINAL CODE (changes Only)
func main() {
allFiles := expandDirectory(dirname)
}
func expandDirectory(currentDirectory string) []os.FileInfo {
allFiles := make([]os.FileInfo, 0, 5)
....
case mode.IsDir():
....
allFiles = append(allFiles, expandDirectory(filepath)...)
....
return allFiles
}
You need to append the result to it:
allFiles = append(allFiles, expandDirectory(filepath, allFiles)...)
Also
allFiles := expandDirectory(dirname, make([]os.FileInfo, 5))
should be:
allFiles := expandDirectory(dirname, make([]os.FileInfo, 0, 5))
Otherwise you will be appending to allFiles and the first 5 items will be empty.
I found some posts on how to decoding json nested objects in go, I tried to apply the answers to my problem, but I only managed to find a partial solution.
My json file look like this:
{
"user":{
"gender":"male",
"age":"21-30",
"id":"80b1ea88-19d7-24e8-52cc-65cf6fb9b380"
},
"trials":{
"0":{"index":0,"word":"WORD 1","Time":3000,"keyboard":true,"train":true,"type":"A"},
"1":{"index":1,"word":"WORD 2","Time":3000,"keyboard":true,"train":true,"type":"A"},
},
"answers":{
"training":[
{"ans":0,"RT":null,"gtAns":"WORD 1","correct":0},
{"ans":0,"RT":null,"gtAns":"WORD 2","correct":0}
],
"test":[
{"ans":0,"RT":null,"gtAns":true,"correct":0},
{"ans":0,"RT":null,"gtAns":true,"correct":0}
]
}
}
Basically I need to parse the information inside it and save them into go structure. With the code below I managed to extract the user information, but it looks too complicated to me and it won't be easy to apply the same thing to the "answers" fields which contains 2 arrays with more than 100 entries each. Here the code I'm using now:
type userDetails struct {
Id string `json:"id"`
Age string `json:"age"`
Gender string `json:"gender"`
}
type jsonRawData map[string]interface {
}
func getJsonContent(r *http.Request) ( userDetails) {
defer r.Body.Close()
jsonBody, err := ioutil.ReadAll(r.Body)
var userDataCurr userDetails
if err != nil {
log.Printf("Couldn't read request body: %s", err)
} else {
var f jsonRawData
err := json.Unmarshal(jsonBody, &f)
if err != nil {
log.Printf("Error unmashalling: %s", err)
} else {
user := f["user"].(map[string]interface{})
userDataCurr.Id = user["id"].(string)
userDataCurr.Gender = user["gender"].(string)
userDataCurr.Age = user["age"].(string)
}
}
return userDataCurr
}
Any suggestions? Thanks a lot!
You're doing it the hard way by using interface{} and not taking advantage of what encoding/json gives you.
I'd do it something like this (note I assumed there was an error with the type of the "gtAns" field and I made it a boolean, you don't give enough information to know what to do with the "RT" field):
package main
import (
"encoding/json"
"fmt"
"io"
"log"
"strconv"
"strings"
)
const input = `{
"user":{
"gender":"male",
"age":"21-30",
"id":"80b1ea88-19d7-24e8-52cc-65cf6fb9b380"
},
"trials":{
"0":{"index":0,"word":"WORD 1","Time":3000,"keyboard":true,"train":true,"type":"A"},
"1":{"index":1,"word":"WORD 2","Time":3000,"keyboard":true,"train":true,"type":"A"}
},
"answers":{
"training":[
{"ans":0,"RT":null,"gtAns":true,"correct":0},
{"ans":0,"RT":null,"gtAns":true,"correct":0}
],
"test":[
{"ans":0,"RT":null,"gtAns":true,"correct":0},
{"ans":0,"RT":null,"gtAns":true,"correct":0}
]
}
}`
type Whatever struct {
User struct {
Gender Gender `json:"gender"`
Age Range `json:"age"`
ID IDString `json:"id"`
} `json:"user"`
Trials map[string]struct {
Index int `json:"index"`
Word string `json:"word"`
Time int // should this be a time.Duration?
Train bool `json:"train"`
Type string `json:"type"`
} `json:"trials"`
Answers map[string][]struct {
Answer int `json:"ans"`
RT json.RawMessage // ??? what type is this
GotAnswer bool `json:"gtAns"`
Correct int `json:"correct"`
} `json:"answers"`
}
// Using some custom types to show custom marshalling:
type IDString string // TODO custom unmarshal and format/error checking
type Gender int
const (
Male Gender = iota
Female
)
func (g *Gender) UnmarshalJSON(b []byte) error {
var s string
err := json.Unmarshal(b, &s)
if err != nil {
return err
}
switch strings.ToLower(s) {
case "male":
*g = Male
case "female":
*g = Female
default:
return fmt.Errorf("invalid gender %q", s)
}
return nil
}
func (g Gender) MarshalJSON() ([]byte, error) {
switch g {
case Male:
return []byte(`"male"`), nil
case Female:
return []byte(`"female"`), nil
default:
return nil, fmt.Errorf("invalid gender %v", g)
}
}
type Range struct{ Min, Max int }
func (r *Range) UnmarshalJSON(b []byte) error {
// XXX could be improved
_, err := fmt.Sscanf(string(b), `"%d-%d"`, &r.Min, &r.Max)
return err
}
func (r Range) MarshalJSON() ([]byte, error) {
return []byte(fmt.Sprintf(`"%d-%d"`, r.Min, r.Max)), nil
// Or:
b := make([]byte, 0, 8)
b = append(b, '"')
b = strconv.AppendInt(b, int64(r.Min), 10)
b = append(b, '-')
b = strconv.AppendInt(b, int64(r.Max), 10)
b = append(b, '"')
return b, nil
}
func fromJSON(r io.Reader) (Whatever, error) {
var x Whatever
dec := json.NewDecoder(r)
err := dec.Decode(&x)
return x, err
}
func main() {
// Use http.Get or whatever to get an io.Reader,
// (e.g. response.Body).
// For playground, substitute a fixed string
r := strings.NewReader(input)
// If you actually had a string or []byte:
// var x Whatever
// err := json.Unmarshal([]byte(input), &x)
x, err := fromJSON(r)
if err != nil {
log.Fatal(err)
}
fmt.Println(x)
fmt.Printf("%+v\n", x)
b, err := json.MarshalIndent(x, "", " ")
if err != nil {
log.Fatal(err)
}
fmt.Printf("Re-marshalled: %s\n", b)
}
Playground
Of course if you want to reuse those sub-types you could pull them out of the "Whatever" type into their own named types.
Also, note the use of a json.Decoder rather than reading in all the data ahead of time. Usually try and avoid any use of ioutil.ReadAll unless you really need all the data at once.