Builds at commandline but fails to build as gae app - google-app-engine

No issues building at commandline:
Darians-MacBook-Pro:gdriveweb darianhickman$ go build helloworld/hello.go
Darians-MacBook-Pro:gdriveweb darianhickman$
Error at locahost:8080/
The Go application could not be built.
(Executed command: /Users/darianhickman/go_appengine/goroot/bin/go-app-builder -app_base /Users/darianhickman/gowork/src/bitbucket.org/darian_hickman/gdriveweb/helloworld -arch 6 -dynamic -goroot /Users/darianhickman/go_appengine/goroot -nobuild_files ^^$ -unsafe -gopath /Users/darianhickman/gowork -binary_name _go_app -extra_imports appengine_internal/init -work_dir /var/folders/fk/wknp5jzn53gbgbml0yn695_m0000gn/T/tmpsHFP6tappengine-go-bin -gcflags -I,/Users/darianhickman/go_appengine/goroot/pkg/darwin_amd64_appengine -ldflags -L,/Users/darianhickman/go_appengine/goroot/pkg/darwin_amd64_appengine hello.go)
/Users/darianhickman/gowork/src/golang.org/x/net/context/ctxhttp/ctxhttp.go:35: req.Cancel undefined (type *http.Request has no field or method Cancel)
2016/05/24 19:39:17 go-app-builder: build timing: 6×6g (469ms total), 0×6l (0 total)
2016/05/24 19:39:17 go-app-builder: failed running 6g: exit status 1
When I research the error
*http.Request has no field or method Cancel
it leads to a bunch of nonapplicable posts about updating to >Go1.5.
Source:
package hello
import (
"encoding/json"
"fmt"
"golang.org/x/net/context"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/drive/v3"
_ "google.golang.org/appengine/urlfetch"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"os/user"
"path/filepath"
)
const (
assetfolder = "0B-zdryEj60U_MXVkajFweXBQWHM"
)
var (
dir *drive.FileList
)
func init() {
http.HandleFunc("/", handler)
ctx := context.Background()
b, err := ioutil.ReadFile("client_secret.json")
if err != nil {
log.Fatalf("Unable to read client secret file: %v", err)
}
// If modifying these scopes, delete your previously saved credentials
// at ~/.credentials/drive-go-quickstart.json
config, err := google.ConfigFromJSON(b, drive.DriveMetadataReadonlyScope)
if err != nil {
log.Fatalf("Unable to parse client secret file to config: %v", err)
}
client := getClient(ctx, config)
srv, err := drive.New(client)
if err != nil {
log.Fatalf("Unable to retrieve drive Client %v", err)
}
dir, err = srv.Files.List().PageSize(10).
Fields("nextPageToken, files(id, name)").Do()
if err != nil {
log.Fatalf("Unable to retrieve files.", err)
}
}
func handler(w http.ResponseWriter, r *http.Request) {
//fmt.Fprint(w, r.RequestURI)
fmt.Fprint(w, "Files:")
if len(dir.Files) > 0 {
for _, i := range dir.Files {
fmt.Fprint(w, "%s (%s)\n", i.Name, i.Id)
}
} else {
fmt.Fprint(w, "No files found.")
}
}
// getClient uses a Context and Config to retrieve a Token
// then generate a Client. It returns the generated Client.
func getClient(ctx context.Context, config *oauth2.Config) *http.Client {
cacheFile, err := tokenCacheFile()
if err != nil {
log.Fatalf("Unable to get path to cached credential file. %v", err)
}
tok, err := tokenFromFile(cacheFile)
if err != nil {
tok = getTokenFromWeb(config)
saveToken(cacheFile, tok)
}
return config.Client(ctx, tok)
}
// getTokenFromWeb uses Config to request a Token.
// It returns the retrieved Token.
func getTokenFromWeb(config *oauth2.Config) *oauth2.Token {
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
fmt.Printf("Go to the following link in your browser then type the "+
"authorization code: \n%v\n", authURL)
var code string
if _, err := fmt.Scan(&code); err != nil {
log.Fatalf("Unable to read authorization code %v", err)
}
tok, err := config.Exchange(oauth2.NoContext, code)
if err != nil {
log.Fatalf("Unable to retrieve token from web %v", err)
}
return tok
}
// tokenCacheFile generates credential file path/filename.
// It returns the generated credential path/filename.
func tokenCacheFile() (string, error) {
usr, err := user.Current()
if err != nil {
return "", err
}
tokenCacheDir := filepath.Join(usr.HomeDir, ".credentials")
os.MkdirAll(tokenCacheDir, 0700)
return filepath.Join(tokenCacheDir,
url.QueryEscape("drive-go-quickstart.json")), err
}
// tokenFromFile retrieves a Token from a given file path.
// It returns the retrieved Token and any read error encountered.
func tokenFromFile(file string) (*oauth2.Token, error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
t := &oauth2.Token{}
err = json.NewDecoder(f).Decode(t)
defer f.Close()
return t, err
}
// saveToken uses a file path to create a file and store the
// token in it.
func saveToken(file string, token *oauth2.Token) {
fmt.Printf("Saving credential file to: %s\n", file)
f, err := os.Create(file)
if err != nil {
log.Fatalf("Unable to cache oauth token: %v", err)
}
defer f.Close()
json.NewEncoder(f).Encode(token)
}
func main() {
ctx := context.Background()
b, err := ioutil.ReadFile("client_secret.json")
if err != nil {
log.Fatalf("Unable to read client secret file: %v", err)
}
// If modifying these scopes, delete your previously saved credentials
// at ~/.credentials/drive-go-quickstart.json
config, err := google.ConfigFromJSON(b, drive.DriveMetadataReadonlyScope)
if err != nil {
log.Fatalf("Unable to parse client secret file to config: %v", err)
}
client := getClient(ctx, config)
srv, err := drive.New(client)
if err != nil {
log.Fatalf("Unable to retrieve drive Client %v", err)
}
r, err := srv.Files.List().PageSize(10).
Fields("nextPageToken, files(id, name)").Do()
if err != nil {
log.Fatalf("Unable to retrieve files.", err)
}
fmt.Println("Files:")
if len(r.Files) > 0 {
for _, i := range r.Files {
fmt.Printf("%s (%s)\n", i.Name, i.Id)
}
} else {
fmt.Print("No files found.")
}
}

I got past this issue by redownloading and reinstalling Go App Engine SDK . My best guess why that worked is that an old version of go was somehow getting included.

Related

Golang with ffmpeg dynamic video Encode

Am trying to encode the videos with Golang ffmpeg. Am not able to get the video file, it shows this error
invalid character '-' in numeric literal
Postman:
This is my code:
package main
import (
"encoding/json"
"fmt"
"net/http"
"github.com/gorilla/mux"
"github.com/xfrr/goffmpeg/transcoder"
)
type Encode struct {
Video string `json:"video"`
}
func encodeFfmpeg(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
fmt.Println("Encodeing Started")
var encode Encode
video := json.NewDecoder(r.Body).Decode(encode)
file, err := os.Open(video)
if err != nil {
log.Fatalln(err)
}
defer file.Close()
buf, err := ioutil.ReadAll(file)
if err != nil {
log.Fatalln(err)
}
cmd := exec.Command("ffmpeg",
"-i", "pipe:0", // take stdin as input
"-c:a", "libmp3lame", // use mp3 lame codec
"-f", "avi",
"pipe:1",
)
resultBuffer := bytes.NewBuffer(make([]byte, 5*1024*1024)) // pre allocate 5MiB buffer
cmd.Stderr = os.Stderr // bind log stream to stderr
cmd.Stdout = resultBuffer // stdout result will be written here
stdin, err := cmd.StdinPipe() // Open stdin pipe
if err != nil {
log.Fatalln(err)
}
err = cmd.Start() // Start a process on another goroutine
if err != nil {
log.Fatalln(err)
}
_, err = stdin.Write(buf) // pump audio data to stdin pipe
if err != nil {
log.Fatalln(err)
}
err = stdin.Close() // close the stdin, or ffmpeg will wait forever
if err != nil {
log.Fatalln(err)
}
err = cmd.Wait() // wait until ffmpeg finish
if err != nil {
log.Fatalln(err)
}
outputFile, err := os.Create(encodeFile) // create new file
if err != nil {
log.Fatalln(err)
}
defer outputFile.Close()
_, err = outputFile.Write(resultBuffer.Bytes()) // write result buffer to file
if err != nil {
log.Fatalln(err)
}
json.NewEncoder(w).Encode(map[string]interface{}{
"status": "success",
"statusCode": 200,
"data": "Successfully Encoded file",
})
}
func main() {
router := mux.NewRouter()
router.HandleFunc("/encode", encodeFfmpeg).Methods("POST")
//router.HandleFunc("/rtsp", rtsp).Methods("POST")
// config port
fmt.Printf("Starting server at 8080 \n")
http.ListenAndServe(":8080", router)
}
video.mp4 file not showing in request body. Please help me to solve this issue
json.NewDecoder(r.Body).Decode(encode) returns error not video and another thing is that you send your data in form so you can access file from form like:
file, header, err := r.FormFile("video")

Record inserted twice into database

I have code in Go like below :
package main
import (
"database/sql"
"log"
_ "github.com/lib/pq"
)
const (
insertLoginSQL = `insert into Logins(id, name,password) values($1, $2, $3)`
)
func main() {
db, err := sql.Open("postgres", "user=postgres password=admin dbname=Quality sslmode=disable")
if err != nil {
log.Fatal(err)
}
defer db.Close()
if err := Insert(db); err != nil {
log.Println("error with double insert", err)
}
}
func Insert(db *sql.DB) error {
tx, err := db.Begin()
if err != nil {
return err
}
stmt, err := tx.Prepare(insertLoginSQL)
if err != nil {
return err
}
defer stmt.Close()
if _, err := stmt.Exec(10, "user","pwd"); err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
When I run above code, records inserted twice in database. Can someone let me know why duplicate records inserted? Any issue with this code?
Probably commit is done twice. First time by some of previous operations like stmt.exec and second time when tx.Commit() executed.

iterate over a large number of entities on appengine with go

On app engine I have a large number of entities of a particular kind.
I want to run a function on each entity (e.g. edit the entity or copy it)
I would do this in a taskqueue but a taskqueue is limited to 10 minutes runtime and each function call is prone to many kinds of errors. What is the best way to do this?
Here's my solution although I'm hoping someone out there has a better solution. I also wonder if this is prone to fork bombs e.g. if the task runs twice, it will set off two chains of iteration.. ! I'm only using it to iterate a few hundred thousand entities, although the operation on each entity is expensive.
First I create a taskqueue for running each individual function call on an entity one at a time:
queue:
- name: entity-iter
rate: 100/s
max_concurrent_requests: 1
retry_parameters:
task_retry_limit: 3
task_age_limit: 30m
min_backoff_seconds: 200
and then I have an iterate entity method which, given the kind, will call your delay func on each entity with the key.
package sysadmin
import (
"google.golang.org/appengine/datastore"
"golang.org/x/net/context"
"google.golang.org/appengine/log"
"google.golang.org/appengine/delay"
"google.golang.org/appengine/taskqueue"
)
func ForEachEntity(kind string, f *delay.Function) *delay.Function {
var callWithNextKey *delay.Function // func(c context.Context, depth int, cursorString string) error
callWithNextKey = delay.Func("something", func(c context.Context, depth int, cursorString string) error {
q := datastore.NewQuery(kind).KeysOnly()
if cursorString != "" {
if curs, err := datastore.DecodeCursor(cursorString); err != nil {
log.Errorf(c, "error decoding cursor %v", err)
return err
} else {
q = q.Start(curs)
}
}
it := q.Run(c)
if key, err := it.Next(nil); err != nil {
if err == datastore.Done {
log.Infof(c, "Done %v", err)
return nil
}
log.Errorf(c, "datastore error %v", err)
return err
} else {
curs, _ := it.Cursor()
if t, err := f.Task(key); err != nil {
return err
} else if _, err = taskqueue.Add(c, t, "entity-iter"); err != nil {
log.Errorf(c, "error %v", err)
return err
}
if depth - 1 > 0 {
if err := callWithNextKey.Call(c, depth - 1, curs.String()); err != nil {
log.Errorf(c, "error2 %v", err)
return err
}
}
}
return nil
})
return callWithNextKey
}
example usage:
var DoCopyCourse = delay.Func("something2", CopyCourse)
var DoCopyCourses = ForEachEntity("Course", DoCopyCourse)
func CopyCourses(c context.Context) {
//sharedmodels.MakeMockCourses(c)
DoCopyCourses.Call(c, 9999999, "")
}

Golang connection to angularjs

How do I connect a backend server in go lang to angularjs? I know angularjs communicates with go via $http or $resource services but what part of the go code links communicates with angular once all the data structs have been made? Would this be the encoded/marshalled json or do we create some kind of route...
I am a newbie looking to start this project after studying angular and go but this is the part i don't understand - what is the end point from go that angulars $resource or $http service links with?
Create a http endpoint that reads json string, unmarshalls it, does some logic with it and writes a json string to the response. E.g:
func main() {
http.HandleFunc("/api/", apiHandler)
logInfo(fmt.Sprintf("Starting server on port %d", serverPort))
err := http.ListenAndServe(fmt.Sprintf(":%d", serverPort), nil)
if err != nil {
logError(err)
}
}
func apiHandler(w http.ResponseWriter, r *http.Request) {
//ensure its a post
method := r.Method
if method != "POST" {
fail(w, "Invalid http method")
return
}
requestData, err := ioutil.ReadAll(r.Body)
if err != nil {
fail(w, err.Error())
return
}
respData, err := doBusinessLogicWithData(requestData)
if err != nil {
fail(w, err.Error())
return
}
respJSONBytes, err := json.Marshal(respData)
if err != nil {
fail(w, err.Error())
return
}
fmt.Fprintln(w, string(respJSONBytes))
}
func fail(w http.ResponseWriter, message string) {
result := &struct {
Success bool
Message string
}{
Success: false,
Message: message,
}
resultJSONBytes, err := json.Marshal(result)
if err != nil {
logError(err)
fmt.Fprintln(w, "Unable to generate result")
return
}
fmt.Fprintln(w, string(resultJSONBytes))
}

blobstore.ParseUpload acts differently in dev server and deployment

I'm trying to send a multipart/form with both a file and an access token,
it works fine with the dev server, but the exact same post to AppEngine deployment result in a different received token string (I can see that its length is a longer. 938 chars when its supposed to be 902).
I'm actually executing the exact same POST request:
curl -X POST --form "token=<ACCESS_TOKEN>" --form "file=#myfile.jpg" http://upload_url
the upload response handler:
c := appengine.NewContext(r)
blobs, values, err := blobstore.ParseUpload(r)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
files := blobs["file"]
if len(files) == 0 {
fmt.Fprintln(w, "No file uploaded")
return
}
token := values.Get("token")
EDIT: I tried to simply create an endpoint for posting the token and printing its length, which returns the correct length.. what am I doing wrong ?
func t(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "%d", len(r.FormValue("token")))
}
EDIT2: when I print the received token from the AppEngine deployment I get something like:
eyJhbGciOiJSUzI1NiIsImtpZCI6ImZjZmQ4NGYxZGZhN2NiODUyMTg4MDFkNDRjNzYwNDFmMzB=
lMzg2OGIifQ.eyJpc3MiOiJhY2NvdW50cy5nb29nbGUuY29tIiwiYXVkIjoiMjEwMTAyMTk5NDI=
4LmFwcHMuZ29vZ2xldXNlcmNvbnRlbnQuY29tIiwidG9rZW5faGFzaCI6IklQMmduQjFsZGMwTE=
VEdVg5LWlZa2ciLCJhdF9oYXNoIjoiSVAyZ25CMWxkYzBMRUR1WDktaVlrZyIsImlkIjoiMTA5O=
.
.
it has line breaks... for some reason the dev server doesn't behave like that and doesn't split the lines.
how can I get the original string or stop this behavior ?
How about using multiplart.Reader?
c := appengine.NewContext(r)
if r.Method != "POST" {
http.Error(w, "invalid request", 400)
return
}
ct := r.Header.Get("Content-Type")
if strings.SplitN(ct, ";", 2)[0] != "multipart/form-data" {
http.Error(w, "invalid request", 40400)
return
}
_, params, err := mime.ParseMediaType(ct)
if err != nil {
http.Error(w, "invalid request", 400)
return
}
boundary, ok := params["boundary"]
if !ok {
http.Error(w, "invalid request", 400)
return
}
reader := multipart.NewReader(r.Body, boundary)
var data []byte
for {
part, err := reader.NextPart()
if part == nil || err != nil {
break
}
if part.FormName() != "file" {
continue
}
v := part.Header.Get("Content-Disposition")
if v == "" {
continue
}
d, _, err := mime.ParseMediaType(v)
if err != nil {
continue
}
if d != "form-data" {
continue
}
data, _ = ioutil.ReadAll(part)
// do something using data
}

Resources