I'm trying to send a multipart/form with both a file and an access token,
it works fine with the dev server, but the exact same post to AppEngine deployment result in a different received token string (I can see that its length is a longer. 938 chars when its supposed to be 902).
I'm actually executing the exact same POST request:
curl -X POST --form "token=<ACCESS_TOKEN>" --form "file=#myfile.jpg" http://upload_url
the upload response handler:
c := appengine.NewContext(r)
blobs, values, err := blobstore.ParseUpload(r)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
files := blobs["file"]
if len(files) == 0 {
fmt.Fprintln(w, "No file uploaded")
return
}
token := values.Get("token")
EDIT: I tried to simply create an endpoint for posting the token and printing its length, which returns the correct length.. what am I doing wrong ?
func t(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "%d", len(r.FormValue("token")))
}
EDIT2: when I print the received token from the AppEngine deployment I get something like:
eyJhbGciOiJSUzI1NiIsImtpZCI6ImZjZmQ4NGYxZGZhN2NiODUyMTg4MDFkNDRjNzYwNDFmMzB=
lMzg2OGIifQ.eyJpc3MiOiJhY2NvdW50cy5nb29nbGUuY29tIiwiYXVkIjoiMjEwMTAyMTk5NDI=
4LmFwcHMuZ29vZ2xldXNlcmNvbnRlbnQuY29tIiwidG9rZW5faGFzaCI6IklQMmduQjFsZGMwTE=
VEdVg5LWlZa2ciLCJhdF9oYXNoIjoiSVAyZ25CMWxkYzBMRUR1WDktaVlrZyIsImlkIjoiMTA5O=
.
.
it has line breaks... for some reason the dev server doesn't behave like that and doesn't split the lines.
how can I get the original string or stop this behavior ?
How about using multiplart.Reader?
c := appengine.NewContext(r)
if r.Method != "POST" {
http.Error(w, "invalid request", 400)
return
}
ct := r.Header.Get("Content-Type")
if strings.SplitN(ct, ";", 2)[0] != "multipart/form-data" {
http.Error(w, "invalid request", 40400)
return
}
_, params, err := mime.ParseMediaType(ct)
if err != nil {
http.Error(w, "invalid request", 400)
return
}
boundary, ok := params["boundary"]
if !ok {
http.Error(w, "invalid request", 400)
return
}
reader := multipart.NewReader(r.Body, boundary)
var data []byte
for {
part, err := reader.NextPart()
if part == nil || err != nil {
break
}
if part.FormName() != "file" {
continue
}
v := part.Header.Get("Content-Disposition")
if v == "" {
continue
}
d, _, err := mime.ParseMediaType(v)
if err != nil {
continue
}
if d != "form-data" {
continue
}
data, _ = ioutil.ReadAll(part)
// do something using data
}
Related
I'm working on GRPC stream, In server side, I receive the multiple byte inside the for loop, I want to merge in a single array of byte (I tried append method but not use), Here I have attached my sample code. Any one guide me.
Example code
func (s *ServerGRPC) Upload(stream pb.GuploadService_UploadServer) (err error) {
for {
resp, err := stream.Recv()
if err != nil {
if err == io.EOF {
goto END
}
err = errors.Wrapf(err,
"failed unexpectadely while reading chunks from stream")
return err
}
for _, result := range resp.Content {
fmt.Println("result ====>>>", result)
//Actual Output
//result ====>>> 136
//result ====>>> 84
//result ====>>> 232
//result ====>>> 12
//Expectation
//result ===> [136 84 232 12]
}
}
s.logger.Info().Msg("upload received")
END:
err = stream.SendAndClose(&pb.UploadStatus{
Message: "Upload received with success",
Code: pb.UploadStatusCode_Ok,
})
if err != nil {
err = errors.Wrapf(err,
"failed to send status code")
return
}
return
}
You are appending to Sample but printing req.Content
There is nothing wrong with merging slices. if just have to print Sample to see merged result.
func (s *ServerGRPC) Upload(stream pb.GuploadService_UploadServer) (err error) {
var respBytes []byte
for {
resp, err := stream.Recv()
if err != nil {
if err == io.EOF {
goto END // you can use break here
}
err = errors.Wrapf(err,
"failed unexpectadely while reading chunks from stream")
return err
}
for _, result := range resp.Content {
fmt.Println("result ====>>>", result)
respBytes = append(respBytes, result)
}
}
s.logger.Info().Msg("upload received")
// print respBytes here
END:
err = stream.SendAndClose(&pb.UploadStatus{
Message: "Upload received with success",
Code: pb.UploadStatusCode_Ok,
})
if err != nil {
err = errors.Wrapf(err,
"failed to send status code")
return
}
return
}
No issues building at commandline:
Darians-MacBook-Pro:gdriveweb darianhickman$ go build helloworld/hello.go
Darians-MacBook-Pro:gdriveweb darianhickman$
Error at locahost:8080/
The Go application could not be built.
(Executed command: /Users/darianhickman/go_appengine/goroot/bin/go-app-builder -app_base /Users/darianhickman/gowork/src/bitbucket.org/darian_hickman/gdriveweb/helloworld -arch 6 -dynamic -goroot /Users/darianhickman/go_appengine/goroot -nobuild_files ^^$ -unsafe -gopath /Users/darianhickman/gowork -binary_name _go_app -extra_imports appengine_internal/init -work_dir /var/folders/fk/wknp5jzn53gbgbml0yn695_m0000gn/T/tmpsHFP6tappengine-go-bin -gcflags -I,/Users/darianhickman/go_appengine/goroot/pkg/darwin_amd64_appengine -ldflags -L,/Users/darianhickman/go_appengine/goroot/pkg/darwin_amd64_appengine hello.go)
/Users/darianhickman/gowork/src/golang.org/x/net/context/ctxhttp/ctxhttp.go:35: req.Cancel undefined (type *http.Request has no field or method Cancel)
2016/05/24 19:39:17 go-app-builder: build timing: 6×6g (469ms total), 0×6l (0 total)
2016/05/24 19:39:17 go-app-builder: failed running 6g: exit status 1
When I research the error
*http.Request has no field or method Cancel
it leads to a bunch of nonapplicable posts about updating to >Go1.5.
Source:
package hello
import (
"encoding/json"
"fmt"
"golang.org/x/net/context"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/drive/v3"
_ "google.golang.org/appengine/urlfetch"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"os/user"
"path/filepath"
)
const (
assetfolder = "0B-zdryEj60U_MXVkajFweXBQWHM"
)
var (
dir *drive.FileList
)
func init() {
http.HandleFunc("/", handler)
ctx := context.Background()
b, err := ioutil.ReadFile("client_secret.json")
if err != nil {
log.Fatalf("Unable to read client secret file: %v", err)
}
// If modifying these scopes, delete your previously saved credentials
// at ~/.credentials/drive-go-quickstart.json
config, err := google.ConfigFromJSON(b, drive.DriveMetadataReadonlyScope)
if err != nil {
log.Fatalf("Unable to parse client secret file to config: %v", err)
}
client := getClient(ctx, config)
srv, err := drive.New(client)
if err != nil {
log.Fatalf("Unable to retrieve drive Client %v", err)
}
dir, err = srv.Files.List().PageSize(10).
Fields("nextPageToken, files(id, name)").Do()
if err != nil {
log.Fatalf("Unable to retrieve files.", err)
}
}
func handler(w http.ResponseWriter, r *http.Request) {
//fmt.Fprint(w, r.RequestURI)
fmt.Fprint(w, "Files:")
if len(dir.Files) > 0 {
for _, i := range dir.Files {
fmt.Fprint(w, "%s (%s)\n", i.Name, i.Id)
}
} else {
fmt.Fprint(w, "No files found.")
}
}
// getClient uses a Context and Config to retrieve a Token
// then generate a Client. It returns the generated Client.
func getClient(ctx context.Context, config *oauth2.Config) *http.Client {
cacheFile, err := tokenCacheFile()
if err != nil {
log.Fatalf("Unable to get path to cached credential file. %v", err)
}
tok, err := tokenFromFile(cacheFile)
if err != nil {
tok = getTokenFromWeb(config)
saveToken(cacheFile, tok)
}
return config.Client(ctx, tok)
}
// getTokenFromWeb uses Config to request a Token.
// It returns the retrieved Token.
func getTokenFromWeb(config *oauth2.Config) *oauth2.Token {
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
fmt.Printf("Go to the following link in your browser then type the "+
"authorization code: \n%v\n", authURL)
var code string
if _, err := fmt.Scan(&code); err != nil {
log.Fatalf("Unable to read authorization code %v", err)
}
tok, err := config.Exchange(oauth2.NoContext, code)
if err != nil {
log.Fatalf("Unable to retrieve token from web %v", err)
}
return tok
}
// tokenCacheFile generates credential file path/filename.
// It returns the generated credential path/filename.
func tokenCacheFile() (string, error) {
usr, err := user.Current()
if err != nil {
return "", err
}
tokenCacheDir := filepath.Join(usr.HomeDir, ".credentials")
os.MkdirAll(tokenCacheDir, 0700)
return filepath.Join(tokenCacheDir,
url.QueryEscape("drive-go-quickstart.json")), err
}
// tokenFromFile retrieves a Token from a given file path.
// It returns the retrieved Token and any read error encountered.
func tokenFromFile(file string) (*oauth2.Token, error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
t := &oauth2.Token{}
err = json.NewDecoder(f).Decode(t)
defer f.Close()
return t, err
}
// saveToken uses a file path to create a file and store the
// token in it.
func saveToken(file string, token *oauth2.Token) {
fmt.Printf("Saving credential file to: %s\n", file)
f, err := os.Create(file)
if err != nil {
log.Fatalf("Unable to cache oauth token: %v", err)
}
defer f.Close()
json.NewEncoder(f).Encode(token)
}
func main() {
ctx := context.Background()
b, err := ioutil.ReadFile("client_secret.json")
if err != nil {
log.Fatalf("Unable to read client secret file: %v", err)
}
// If modifying these scopes, delete your previously saved credentials
// at ~/.credentials/drive-go-quickstart.json
config, err := google.ConfigFromJSON(b, drive.DriveMetadataReadonlyScope)
if err != nil {
log.Fatalf("Unable to parse client secret file to config: %v", err)
}
client := getClient(ctx, config)
srv, err := drive.New(client)
if err != nil {
log.Fatalf("Unable to retrieve drive Client %v", err)
}
r, err := srv.Files.List().PageSize(10).
Fields("nextPageToken, files(id, name)").Do()
if err != nil {
log.Fatalf("Unable to retrieve files.", err)
}
fmt.Println("Files:")
if len(r.Files) > 0 {
for _, i := range r.Files {
fmt.Printf("%s (%s)\n", i.Name, i.Id)
}
} else {
fmt.Print("No files found.")
}
}
I got past this issue by redownloading and reinstalling Go App Engine SDK . My best guess why that worked is that an old version of go was somehow getting included.
On app engine I have a large number of entities of a particular kind.
I want to run a function on each entity (e.g. edit the entity or copy it)
I would do this in a taskqueue but a taskqueue is limited to 10 minutes runtime and each function call is prone to many kinds of errors. What is the best way to do this?
Here's my solution although I'm hoping someone out there has a better solution. I also wonder if this is prone to fork bombs e.g. if the task runs twice, it will set off two chains of iteration.. ! I'm only using it to iterate a few hundred thousand entities, although the operation on each entity is expensive.
First I create a taskqueue for running each individual function call on an entity one at a time:
queue:
- name: entity-iter
rate: 100/s
max_concurrent_requests: 1
retry_parameters:
task_retry_limit: 3
task_age_limit: 30m
min_backoff_seconds: 200
and then I have an iterate entity method which, given the kind, will call your delay func on each entity with the key.
package sysadmin
import (
"google.golang.org/appengine/datastore"
"golang.org/x/net/context"
"google.golang.org/appengine/log"
"google.golang.org/appengine/delay"
"google.golang.org/appengine/taskqueue"
)
func ForEachEntity(kind string, f *delay.Function) *delay.Function {
var callWithNextKey *delay.Function // func(c context.Context, depth int, cursorString string) error
callWithNextKey = delay.Func("something", func(c context.Context, depth int, cursorString string) error {
q := datastore.NewQuery(kind).KeysOnly()
if cursorString != "" {
if curs, err := datastore.DecodeCursor(cursorString); err != nil {
log.Errorf(c, "error decoding cursor %v", err)
return err
} else {
q = q.Start(curs)
}
}
it := q.Run(c)
if key, err := it.Next(nil); err != nil {
if err == datastore.Done {
log.Infof(c, "Done %v", err)
return nil
}
log.Errorf(c, "datastore error %v", err)
return err
} else {
curs, _ := it.Cursor()
if t, err := f.Task(key); err != nil {
return err
} else if _, err = taskqueue.Add(c, t, "entity-iter"); err != nil {
log.Errorf(c, "error %v", err)
return err
}
if depth - 1 > 0 {
if err := callWithNextKey.Call(c, depth - 1, curs.String()); err != nil {
log.Errorf(c, "error2 %v", err)
return err
}
}
}
return nil
})
return callWithNextKey
}
example usage:
var DoCopyCourse = delay.Func("something2", CopyCourse)
var DoCopyCourses = ForEachEntity("Course", DoCopyCourse)
func CopyCourses(c context.Context) {
//sharedmodels.MakeMockCourses(c)
DoCopyCourses.Call(c, 9999999, "")
}
How do I connect a backend server in go lang to angularjs? I know angularjs communicates with go via $http or $resource services but what part of the go code links communicates with angular once all the data structs have been made? Would this be the encoded/marshalled json or do we create some kind of route...
I am a newbie looking to start this project after studying angular and go but this is the part i don't understand - what is the end point from go that angulars $resource or $http service links with?
Create a http endpoint that reads json string, unmarshalls it, does some logic with it and writes a json string to the response. E.g:
func main() {
http.HandleFunc("/api/", apiHandler)
logInfo(fmt.Sprintf("Starting server on port %d", serverPort))
err := http.ListenAndServe(fmt.Sprintf(":%d", serverPort), nil)
if err != nil {
logError(err)
}
}
func apiHandler(w http.ResponseWriter, r *http.Request) {
//ensure its a post
method := r.Method
if method != "POST" {
fail(w, "Invalid http method")
return
}
requestData, err := ioutil.ReadAll(r.Body)
if err != nil {
fail(w, err.Error())
return
}
respData, err := doBusinessLogicWithData(requestData)
if err != nil {
fail(w, err.Error())
return
}
respJSONBytes, err := json.Marshal(respData)
if err != nil {
fail(w, err.Error())
return
}
fmt.Fprintln(w, string(respJSONBytes))
}
func fail(w http.ResponseWriter, message string) {
result := &struct {
Success bool
Message string
}{
Success: false,
Message: message,
}
resultJSONBytes, err := json.Marshal(result)
if err != nil {
logError(err)
fmt.Fprintln(w, "Unable to generate result")
return
}
fmt.Fprintln(w, string(resultJSONBytes))
}
For some reason nothing gets saved when the test code below is run. I have other api methods that when run (not through tests, this is just the first test) do save.
When I check the database stats via localhost:8000, it can be seen that nothing is being inserted.
Update: After copying and pasting the code below and wrapping it is GET request handler with some hardcoded data it does save to the database. So this seems like an issue with the testing aetest.Context that is used. I have added the code for the NewTestHandler helper code.
Method to create the context within the tests
func NewTestHandler(handlerFunc func(appengine.Context, http.ResponseWriter, *http.Request)) http.HandlerFunc {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, _ := aetest.NewContext(nil)
handlerFunc(c, w, r)
})
}
Error (update: the key that is generated returns 0 when calling .IntId())
// happens in the .Get() error handling
--- err datastore: internal error: server returned the wrong number of entities
Model
package app
import "time"
type League struct {
Name string `json:"name"`
Location string `json:"location"`
CreatedAt time.Time
}
Code
func (api *LeagueApi) Create(c appengine.Context, w http.ResponseWriter, r *http.Request) {
// data
var league League
json.NewDecoder(r.Body).Decode(&league)
defer r.Body.Close()
// save to db
key := datastore.NewIncompleteKey(c, "leagues", nil)
if _, err := datastore.Put(c, key, &league); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var leagueCheck League
if err := datastore.Get(c, key, &leagueCheck); err != nil {
log.Println("--- err", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// json response
if err := json.NewEncoder(w).Encode(league); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
Test
func Test_LeagueReturnedOnCreate(t *testing.T) {
league := League{Name: "foobar"}
data, _ := json.Marshal(league)
reader := bytes.NewReader(data)
// setup request and writer
r, _ := http.NewRequest("POST", "/leagues", reader)
w := httptest.NewRecorder()
// make request
api := LeagueApi{}
handler := tux.NewTestHandler(api.Create)
handler.ServeHTTP(w, r)
// extract api response
var leagueCheck League
json.NewDecoder(w.Body).Decode(&leagueCheck)
if leagueCheck.Name != "foobar" {
t.Error("should return the league")
}
// ensure the league is in the db
}