I want to run 2 goroutines parallel in App Engine, so that when the first goroutine finish its job, the handler doesn't need to wait the second goroutine - it stops the secend goroutine and returns the result to the client. Is this possible? I tried it with context.WithCancel(), but it didn't work (I use go1.6).
Here is my code:
package mytest
import (
"net/http"
"sync"
"time"
"golang.org/x/net/context"
"google.golang.org/appengine"
"google.golang.org/appengine/log"
"google.golang.org/appengine/urlfetch"
)
func init() {
http.HandleFunc("/test", handlerTest)
http.HandleFunc("/testwait10s", handlerTest10s)
http.HandleFunc("/testwait5s", handlerTest5s)
}
func handlerTest(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
ctx, _ := context.WithTimeout(c, 30*time.Second)
ctx1, ctx1Cancel := context.WithCancel(ctx)
ctx2, ctx2Cancel := context.WithCancel(ctx)
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
log.Infof(ctx1, "Go1 begin ...")
client1 := urlfetch.Client(ctx1)
_, err := client1.Get("http://APP_NAME.appspot.com/testwait5s")
if err != nil {
log.Errorf(ctx1, "Go1 failed: %v", err)
}
ctx2Cancel()
log.Infof(ctx1, "Go1 over ...")
}()
go func() {
defer wg.Done()
log.Infof(ctx2, "Go2 begin ...")
client2 := urlfetch.Client(ctx2)
_, err := client2.Get("http://APP_NAME.appspot.com/testwait10s")
if err != nil {
log.Errorf(ctx2, "Go2 failed %v", err)
}
ctx1Cancel()
log.Infof(ctx2, "Go2 over ...")
}()
wg.Wait()
log.Infof(ctx1, "Go1 and GO2 over")
}
func handlerTest10s(w http.ResponseWriter, r *http.Request) {
time.Sleep(10 * time.Second)
return
}
func handlerTest5s(w http.ResponseWriter, r *http.Request) {
time.Sleep(5 * time.Second)
return
}
Any ideas? Thanks!
Just create a notification channel and send there a signal that one of computations is over and you can proceed without waiting for the other.
func handlerTest(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
ctx, cancel := context.WithTimeout(c, 30*time.Second)
done := make(chan error, 2)
work := func(url, name string) {
log.Infof(ctx, "%s begin ...", name)
client := urlfetch.Client(ctx)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
log.Errorf(ctx, "%s failed: %v", name, err)
done <- err
return
}
req = req.WithContext(ctx)
_, err = client.Do(req)
done <- err
if err != nil {
log.Errorf(ctx, "%s failed: %v", name, err)
return
}
cancel()
log.Infof(ctx, "%s over ...", name)
}
go work("go1", "http://APP_NAME.appspot.com/testwait5s")
go work("go2", "http://APP_NAME.appspot.com/testwait10s")
for i := 0; i < cap(done); i++ {
if err := <-done; err == nil {
log.Infof(ctx, "executed without errors")
return
}
}
log.Error(ctx, "both computations have failed")
}
You can try reducing the value of wg.Add() to wg.Add(1) instead of wg.Add(2).
When one go-routine completes, wg.Done() will reduce the counter value by 1. So, In this case, the WaitGroup (wg) counter value will become ZERO. As a result, wg.Wait() on last line, will not wait for other go-routines to complete.
Note that, if the value of wg counter falls below zero, it will panic the remaining go-routines. So, the go-routines will be exited forcefully.
Related
i am trying to learn some new stuff with GoLang, and got a litlebit stuck, probaly the reason is just that i am not very good at using arrays.
So heres what i want to do:
Make variable.
Download with that variable.
Add ++1 for that variable
Download with added 1
and loop it lets say 10 times.
I am all good with points 1 and two, but little stuck with 3 & 4. :).
all the files come in .pdf, thats why i made that strconv there.
I probaly should make somekind of Loop in main, and call DownloadFile function with some array parameters in there?
package main
import (
"fmt"
"io"
"net/http"
"os"
"strconv"
)
func main() {
url_id := strconv.Itoa(23430815+2)
filename := url_id+".pdf"
fileUrl := "https://someurLid="+url_id
if err := DownloadFile(filename, fileUrl); err != nil {
panic(err)
}
fmt.Println(fileUrl)
}
func DownloadFile(filepath string, url string) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
out, err := os.Create(filepath)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, resp.Body)
return err
}
try this.
package main
import (
"fmt"
"io"
"net/http"
"os"
"strconv"
)
func main() {
url_id_num := 23430815+2
for i := 0; i < 10; i++ {
url_id := strconv.Itoa(url_id_num+i)
filename := url_id+".pdf"
fileUrl := "https://someurLid="+url_id
if err := DownloadFile(filename, fileUrl); err != nil {
panic(err)
}
fmt.Println(fileUrl)
}
}
func DownloadFile(filepath string, url string) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
out, err := os.Create(filepath)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, resp.Body)
return err
}
Cerise Limón gave the answer and thats thats how it worked out.
arr := make([]uint8, 3) //How many times it loops
url_id := 23430815 //Starting from id, filename
for range arr {
filename := strconv.Itoa(url_id)+".pdf"
fileUrl := "https://someurl?id="+strconv.Itoa(url_id)
if err := DownloadFile(filename, fileUrl); err != nil {
panic(err)
}
fmt.Println(fileUrl)
url_id++
}
Thank you for pointing out where i should start! :).
I have code in Go like below :
package main
import (
"database/sql"
"log"
_ "github.com/lib/pq"
)
const (
insertLoginSQL = `insert into Logins(id, name,password) values($1, $2, $3)`
)
func main() {
db, err := sql.Open("postgres", "user=postgres password=admin dbname=Quality sslmode=disable")
if err != nil {
log.Fatal(err)
}
defer db.Close()
if err := Insert(db); err != nil {
log.Println("error with double insert", err)
}
}
func Insert(db *sql.DB) error {
tx, err := db.Begin()
if err != nil {
return err
}
stmt, err := tx.Prepare(insertLoginSQL)
if err != nil {
return err
}
defer stmt.Close()
if _, err := stmt.Exec(10, "user","pwd"); err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
When I run above code, records inserted twice in database. Can someone let me know why duplicate records inserted? Any issue with this code?
Probably commit is done twice. First time by some of previous operations like stmt.exec and second time when tx.Commit() executed.
No issues building at commandline:
Darians-MacBook-Pro:gdriveweb darianhickman$ go build helloworld/hello.go
Darians-MacBook-Pro:gdriveweb darianhickman$
Error at locahost:8080/
The Go application could not be built.
(Executed command: /Users/darianhickman/go_appengine/goroot/bin/go-app-builder -app_base /Users/darianhickman/gowork/src/bitbucket.org/darian_hickman/gdriveweb/helloworld -arch 6 -dynamic -goroot /Users/darianhickman/go_appengine/goroot -nobuild_files ^^$ -unsafe -gopath /Users/darianhickman/gowork -binary_name _go_app -extra_imports appengine_internal/init -work_dir /var/folders/fk/wknp5jzn53gbgbml0yn695_m0000gn/T/tmpsHFP6tappengine-go-bin -gcflags -I,/Users/darianhickman/go_appengine/goroot/pkg/darwin_amd64_appengine -ldflags -L,/Users/darianhickman/go_appengine/goroot/pkg/darwin_amd64_appengine hello.go)
/Users/darianhickman/gowork/src/golang.org/x/net/context/ctxhttp/ctxhttp.go:35: req.Cancel undefined (type *http.Request has no field or method Cancel)
2016/05/24 19:39:17 go-app-builder: build timing: 6×6g (469ms total), 0×6l (0 total)
2016/05/24 19:39:17 go-app-builder: failed running 6g: exit status 1
When I research the error
*http.Request has no field or method Cancel
it leads to a bunch of nonapplicable posts about updating to >Go1.5.
Source:
package hello
import (
"encoding/json"
"fmt"
"golang.org/x/net/context"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/drive/v3"
_ "google.golang.org/appengine/urlfetch"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"os/user"
"path/filepath"
)
const (
assetfolder = "0B-zdryEj60U_MXVkajFweXBQWHM"
)
var (
dir *drive.FileList
)
func init() {
http.HandleFunc("/", handler)
ctx := context.Background()
b, err := ioutil.ReadFile("client_secret.json")
if err != nil {
log.Fatalf("Unable to read client secret file: %v", err)
}
// If modifying these scopes, delete your previously saved credentials
// at ~/.credentials/drive-go-quickstart.json
config, err := google.ConfigFromJSON(b, drive.DriveMetadataReadonlyScope)
if err != nil {
log.Fatalf("Unable to parse client secret file to config: %v", err)
}
client := getClient(ctx, config)
srv, err := drive.New(client)
if err != nil {
log.Fatalf("Unable to retrieve drive Client %v", err)
}
dir, err = srv.Files.List().PageSize(10).
Fields("nextPageToken, files(id, name)").Do()
if err != nil {
log.Fatalf("Unable to retrieve files.", err)
}
}
func handler(w http.ResponseWriter, r *http.Request) {
//fmt.Fprint(w, r.RequestURI)
fmt.Fprint(w, "Files:")
if len(dir.Files) > 0 {
for _, i := range dir.Files {
fmt.Fprint(w, "%s (%s)\n", i.Name, i.Id)
}
} else {
fmt.Fprint(w, "No files found.")
}
}
// getClient uses a Context and Config to retrieve a Token
// then generate a Client. It returns the generated Client.
func getClient(ctx context.Context, config *oauth2.Config) *http.Client {
cacheFile, err := tokenCacheFile()
if err != nil {
log.Fatalf("Unable to get path to cached credential file. %v", err)
}
tok, err := tokenFromFile(cacheFile)
if err != nil {
tok = getTokenFromWeb(config)
saveToken(cacheFile, tok)
}
return config.Client(ctx, tok)
}
// getTokenFromWeb uses Config to request a Token.
// It returns the retrieved Token.
func getTokenFromWeb(config *oauth2.Config) *oauth2.Token {
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
fmt.Printf("Go to the following link in your browser then type the "+
"authorization code: \n%v\n", authURL)
var code string
if _, err := fmt.Scan(&code); err != nil {
log.Fatalf("Unable to read authorization code %v", err)
}
tok, err := config.Exchange(oauth2.NoContext, code)
if err != nil {
log.Fatalf("Unable to retrieve token from web %v", err)
}
return tok
}
// tokenCacheFile generates credential file path/filename.
// It returns the generated credential path/filename.
func tokenCacheFile() (string, error) {
usr, err := user.Current()
if err != nil {
return "", err
}
tokenCacheDir := filepath.Join(usr.HomeDir, ".credentials")
os.MkdirAll(tokenCacheDir, 0700)
return filepath.Join(tokenCacheDir,
url.QueryEscape("drive-go-quickstart.json")), err
}
// tokenFromFile retrieves a Token from a given file path.
// It returns the retrieved Token and any read error encountered.
func tokenFromFile(file string) (*oauth2.Token, error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
t := &oauth2.Token{}
err = json.NewDecoder(f).Decode(t)
defer f.Close()
return t, err
}
// saveToken uses a file path to create a file and store the
// token in it.
func saveToken(file string, token *oauth2.Token) {
fmt.Printf("Saving credential file to: %s\n", file)
f, err := os.Create(file)
if err != nil {
log.Fatalf("Unable to cache oauth token: %v", err)
}
defer f.Close()
json.NewEncoder(f).Encode(token)
}
func main() {
ctx := context.Background()
b, err := ioutil.ReadFile("client_secret.json")
if err != nil {
log.Fatalf("Unable to read client secret file: %v", err)
}
// If modifying these scopes, delete your previously saved credentials
// at ~/.credentials/drive-go-quickstart.json
config, err := google.ConfigFromJSON(b, drive.DriveMetadataReadonlyScope)
if err != nil {
log.Fatalf("Unable to parse client secret file to config: %v", err)
}
client := getClient(ctx, config)
srv, err := drive.New(client)
if err != nil {
log.Fatalf("Unable to retrieve drive Client %v", err)
}
r, err := srv.Files.List().PageSize(10).
Fields("nextPageToken, files(id, name)").Do()
if err != nil {
log.Fatalf("Unable to retrieve files.", err)
}
fmt.Println("Files:")
if len(r.Files) > 0 {
for _, i := range r.Files {
fmt.Printf("%s (%s)\n", i.Name, i.Id)
}
} else {
fmt.Print("No files found.")
}
}
I got past this issue by redownloading and reinstalling Go App Engine SDK . My best guess why that worked is that an old version of go was somehow getting included.
For some reason nothing gets saved when the test code below is run. I have other api methods that when run (not through tests, this is just the first test) do save.
When I check the database stats via localhost:8000, it can be seen that nothing is being inserted.
Update: After copying and pasting the code below and wrapping it is GET request handler with some hardcoded data it does save to the database. So this seems like an issue with the testing aetest.Context that is used. I have added the code for the NewTestHandler helper code.
Method to create the context within the tests
func NewTestHandler(handlerFunc func(appengine.Context, http.ResponseWriter, *http.Request)) http.HandlerFunc {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, _ := aetest.NewContext(nil)
handlerFunc(c, w, r)
})
}
Error (update: the key that is generated returns 0 when calling .IntId())
// happens in the .Get() error handling
--- err datastore: internal error: server returned the wrong number of entities
Model
package app
import "time"
type League struct {
Name string `json:"name"`
Location string `json:"location"`
CreatedAt time.Time
}
Code
func (api *LeagueApi) Create(c appengine.Context, w http.ResponseWriter, r *http.Request) {
// data
var league League
json.NewDecoder(r.Body).Decode(&league)
defer r.Body.Close()
// save to db
key := datastore.NewIncompleteKey(c, "leagues", nil)
if _, err := datastore.Put(c, key, &league); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var leagueCheck League
if err := datastore.Get(c, key, &leagueCheck); err != nil {
log.Println("--- err", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// json response
if err := json.NewEncoder(w).Encode(league); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
Test
func Test_LeagueReturnedOnCreate(t *testing.T) {
league := League{Name: "foobar"}
data, _ := json.Marshal(league)
reader := bytes.NewReader(data)
// setup request and writer
r, _ := http.NewRequest("POST", "/leagues", reader)
w := httptest.NewRecorder()
// make request
api := LeagueApi{}
handler := tux.NewTestHandler(api.Create)
handler.ServeHTTP(w, r)
// extract api response
var leagueCheck League
json.NewDecoder(w.Body).Decode(&leagueCheck)
if leagueCheck.Name != "foobar" {
t.Error("should return the league")
}
// ensure the league is in the db
}
I am trying to learn Go with GAE.
I have created 2 handlers. One for saving an object to datastore and the other retrieve it and output to screen. The problem is that when i retrieve the UserAccount object from datastore, every values inside the object are gone.
Any help would be appreciate.
Output:
a/c count: 2
val: core.UserAccount{idString:"", deviceId:""}
val: core.UserAccount{idString:"", deviceId:""}
type UserAccount struct {
idString string
deviceId string
}
func create_account(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
idstr := "ABCDEFGH"
devId := r.FormValue("deviceId")
newAccount := UserAccount{ idString: idstr, deviceId: devId,}
key := datastore.NewIncompleteKey(c, "UserAccount", nil)
_, err := datastore.Put(c, key, &newAccount)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
fmt.Fprintf(w, "val: %#v \n", newAccount)
}
func get_info(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
q := datastore.NewQuery("UserAccount")
accounts := make([]UserAccount, 0, 10)
if _, err := q.GetAll(c, &accounts); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
fmt.Fprintf(w, "a/c count: %v \n", len(accounts))
for i := 0; i < len(accounts); i++ {
fmt.Fprintf(w, "val: %#v \n", accounts[i])
}
}
If the datastore API uses reflection, which I presume it does, it cannot access struct fields that aren't exported, i.e. field names that do not begin with a capital letter.
Export them and it should work.