I'm trying to have Go submit a form on a webpage for me to simulate a login. From there I'm trying to use the cookies to keep a persistent session for one more call to a sub-page.
I'm able to successfully do the log in with no issues, I'm just having issues catching the cookies being set by the returning server. I'm wondering if it's because their login script does several redirects? (I am getting an output).
Any ideas why I'm not catching the cookies being returned?
Here is the code I'm using:
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"sync"
)
type Jar struct {
lk sync.Mutex
cookies map[string][]*http.Cookie
}
var CookieJar *Jar
func NewJar() *Jar {
jar := new(Jar)
jar.cookies = make(map[string][]*http.Cookie)
return jar
}
// SetCookies handles the receipt of the cookies in a reply for the
// given URL. It may or may not choose to save the cookies, depending
// on the jar's policy and implementation.
func (jar *Jar) SetCookies(u *url.URL, cookies []*http.Cookie) {
jar.lk.Lock()
jar.cookies[u.Host] = cookies
jar.lk.Unlock()
}
// Cookies returns the cookies to send in a request for the given URL.
// It is up to the implementation to honor the standard cookie use
// restrictions such as in RFC 6265.
func (jar *Jar) Cookies(u *url.URL) []*http.Cookie {
return jar.cookies[u.Host]
}
func NewClient() *http.Client {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: false},
}
CookieJar = NewJar()
client := &http.Client{
Transport: tr,
CheckRedirect: nil,
Jar: CookieJar,
}
return client
}
func Login() {
client := NewClient()
api := "https://www.statuscake.com/App/"
uri, _ := url.Parse("https://www.statuscake.com")
fmt.Printf("uri: %s\n", uri)
values := url.Values{}
values.Add("username", username)
values.Add("password", password)
values.Add("Login", "yes")
values.Add("redirect", "")
str := values.Encode()
req, err := http.NewRequest("POST", api, strings.NewReader(str))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "text/html")
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36")
cookies := CookieJar.Cookies(uri)
for i := 0; i < len(cookies); i++ {
fmt.Printf("Cookie[%d]: %s", i, cookies[i])
req.AddCookie(cookies[i])
}
resp, err := client.Do(req)
if err != nil {
panic(err)
}
fmt.Printf("Response: %v\n", resp)
fmt.Printf("Response.Cookies: %v\n", resp.Cookies())
cookies = resp.Cookies()
CookieJar.SetCookies(uri, cookies)
defer resp.Body.Close()
if resp.StatusCode == 200 {
fmt.Printf("\n\n-----\n")
fmt.Println("HTTP Code: ", resp.StatusCode)
fmt.Println("Response Cookies: ", resp.Cookies())
fmt.Println("Request Headers: ", req.Header)
fmt.Println("Request Cookies: ", req.Cookies())
fmt.Println("Response Headers: ", resp.Header)
fmt.Printf("-----\n\n")
}
}
Not sure where the problem really is, but two notes:
There is no need to populate the request manually with cookies if your Client has a working Jar. The client should handle all cookie stuff transparently for you: It extracts cookies from the response and store in the jar and populates the response with cookies from the jar (even during redirects). Maybe this overwrites your manually set cookies.
Do not roll your own CookieJar implementation. Cookie handling is awful (you may believe me). Just use http://golang.org/pkg/net/http/cookiejar/ maybe in combination with code.google.com/p/go.net/publicsuffix
Related
I'm trying to push the message to google pub-sub asynchronously through goroutine but I'm facing below error
panic: not an App Engine context
I'm using mux and have an api handler
n = 1 million
func apihandler(w http.ResponseWriter, r *http.Request) {
go createuniquecodes(n)
return "request running in background"
}
func createuniquecodes(n) {
c := make(chan string)
go createuniquecodes(c, n)
for val := range c {
publishtopubsub(val)
}
}
func createuniquecodes(n) {
for i := 0; i < n; i++ {
uniquecode := some random string
// publish to channel and pubsub
c <- uniquecode
}
close(c)
}
func publishuq(msg string) error {
ctx := context.Background()
client, err := pubsub.NewClient(ctx, projectId)
if err != nil {
log.Fatalf("Could not create pubsub Client: %v", err)
}
t := client.Topic(topicName)
result := t.Publish(ctx, &pubsub.Message{
Data: []byte(msg),
})
id, err := result.Get(ctx)
if err != nil {
return err
}
fmt.Printf("Published a message; msg ID: %v\n", id)
return nil
}
Please note that I need to generate 5 million unique codes,
How will I define a context in go routine since I'm doing everything asynchronously
I assume you're using the App Engine standard (not flexible) environment. Please note that a "request handler (apihandler in your case) has a limited amount of time to generate and return a response to a request, typically around 60 seconds. Once the deadline has been reached, the request handler is interrupted".
You're trying to "break out" of the request when calling go createuniquecodes(n) and then ctx := context.Background() down the line is what panics with not an App Engine context. You could technically use NewContext(req *http.Request) to derive a valid context from the original context, but again, you'd only have 60s before your request times out.
Please have a look at TaskQueues, as they " let applications perform work, called tasks, asynchronously outside of a user request."
I've written a test code to list buckets from Google Cloud Storage through the Cloud Storage API, however I'm getting the error below when I run the code:
googleapi: Error 403: Forbidden, forbiddenFinished
I've checked the permissions and the appengine service account has access to the buckets and both the appengine app and cloud storage bucket are on the same project.
This is my sample code:
package src
import (
"fmt"
"net/http"
"golang.org/x/oauth2/google"
storage "google.golang.org/api/storage/v1"
appengine "google.golang.org/appengine"
)
func init() {
http.HandleFunc("/", index)
}
func ListBuckets(r *http.Request, projectID string) ([]*storage.Bucket, error) {
ctx := appengine.NewContext(r)
client, err := google.DefaultClient(ctx, storage.DevstorageReadOnlyScope)
if err != nil {
return nil, err
}
service, err := storage.New(client)
if err != nil {
return nil, err
}
buckets, err := service.Buckets.List(projectID).Do()
if err != nil {
return nil, err
}
return buckets.Items, nil
}
func index(w http.ResponseWriter, r *http.Request) {
r.Header.Set("x-goog-project-id", "theIdProvidedByTheAPI")
bucket, err := ListBuckets(r,"myProject")
if err != nil {
fmt.Fprint(w,err.Error())
}
for i:=range bucket {
fmt.Fprint(w,bucket[i].Name)
}
fmt.Fprint(w,"\n","Finished.")
}
And this is the yaml file:
application: myProject
version: alpha-001
runtime: go
api_version: go1
handlers:
- url: /
script: _go_app
The error message is nor helpful as it does not provide much useful information. I just can't figure out what I'm missing.
I've had issues with the google.DefaultClient method in the past. Here's a more explicit strategy for configuring the storage client object that might be of use for you:
httpClient = &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, scopes...),
Base: &urlfetch.Transport{Context: ctx},
},
}
service, err := storage.New(httpClient)
if err != nil {
return nil, err
}
However, I'm not familiar with the forbiddenFinished error message, which may indicate that the issue lies elsewhere.
Additionally, if you don't have a specific reason for using the autogenerated google.golang.org/api/storage/v1 library, I'd recommend using the higher level interface, which can be found at cloud.google.com/go/storage. Here's the go doc for it:
https://godoc.org/cloud.google.com/go/storage
Full code example here: https://github.com/crosbymichael/not-dockers-ui/commit/15d133324d22e84e3c2839d19b112a96ced4dd66
I've wrapped the handler with CSRF.Protect(), Angular is finding the _gorilla_csrf cookie and passing it back as the X-CSRF-Token header, but I always receive invalid token errors on non-GET requests:
Forbidden - CSRF token invalid
Am I trying to use the gorilla/csrf incorrectly?
Edit: Simplified example of what I'm trying to do:
var (
mux = http.NewServeMux()
fileHandler = http.FileServer(http.Dir(""))
h http.Handler
CSRF = csrf.Protect(
[]byte("32-byte-long-auth-key"),
csrf.HttpOnly(false),
csrf.Secure(false),
)
)
u, err := url.Parse("http://host/api")
if err != nil {
log.Fatal(err)
}
h = httputil.NewSingleHostReverseProxy(u)
mux.Handle("/dockerapi/", http.StripPrefix("/dockerapi", h))
mux.Handle("/", fileHandler)
handler := CSRF(mux)
if err := http.ListenAndServe(*addr, handler); err != nil {
log.Fatal(err)
}
We serve static files from the server directly, and API requests pass through to an external backend. The app is an Angular SPA that often has multiple requests in flight to the API, but typically only has one non-GET request active at any time.
I try to connect to drive with a service account.
Actually I have
c := appengine.NewContext(r)
key, err := ioutil.ReadFile("key/key.pem")
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
c.Errorf("Pem file not found")
return
}
config := &jwt.Config{
Email: "xxx#developer.gserviceaccount.com",
PrivateKey: key,
Scopes: []string{
"https://www.googleapis.com/auth/drive",
},
TokenURL: google.JWTTokenURL,
}
client := config.Client(oauth2.NoContext)
service, err := drive.New(client)
if (err != nil) {
w.WriteHeader(http.StatusInternalServerError)
c.Errorf("Service connection not works")
return
}
about, err := service.About.Get().Do()
if (err != nil) {
w.WriteHeader(http.StatusInternalServerError)
c.Errorf(err.Error())
return
}
c.Infof(about.Name)
That I found here : https://github.com/golang/oauth2/blob/master/google/example_test.go
Of course it doesn't work, I have to use urlfetch, but I don't know how...
The error I get is "ERROR: Get https://www.googleapis.com/drive/v2/about?alt=json: oauth2: cannot fetch token: Post https://accounts.google.com/o/oauth2/token: not an App Engine context"
How I can do?
Thank you.
There are two Go packages for Google App Engine: appengine and google.golang.org/appengine.
The first one uses appengine.Context that is not compatible with the context.Context used by the oauth2 packages. You need to import the second one to google.golang.org/appengine.
Also, change client := config.Client(oauth2.NoContext) to client := config.Client(c).
I am creating a large database application in Google App Engine Go. Most of my pieces of data are small, so I have no problem storing them in Datastore. However, I know I will run into a few entries that will be a few megabytes big, so I will have to use Blobstore to save them.
Looking at the reference for Blobstore, it appears that the service was mainly intended to be used for files being uploaded to the service. What are the functions I need to call to store arbitrary data in the Blobstore like I would in Datastore? I can already convert the data to []byte and I don't need to index anything in the blob, just to store and fetch it by ID.
There are two ways that you could write files to the blobstore
One is to use a deprecated API documented at the end of the page for the blobstore. Their example code is below.
The approach that they are going to be switching to is storing files in Google cloud storage and serving them via the blobstore.
The other approach would be to simulate a user upload in some fashion. Go has an http client that can send files to be uploaded to web addresses. That would be a hacky way to do it though.
var k appengine.BlobKey
w, err := blobstore.Create(c, "application/octet-stream")
if err != nil {
return k, err
}
_, err = w.Write([]byte("... some data ..."))
if err != nil {
return k, err
}
err = w.Close()
if err != nil {
return k, err
}
return w.Key()
As #yumaikas said, the Files API is deprecated. If this data comes from some sort of a user upload, you should modify the upload form to work with Blobstore Upload URLs (in particular, setting the encoding to multipart/form-data or multipart/mixed and naming all file upload fields file, except the ones that you don't want to be stored in blobstore).
However, if that is not possible (e.g. you don't have control over the user input, or you have to pre-process the data on the server before you store it in Blobstore), then you'll either have to use the deprecated Files API, or upload the data using the URLFetch API.
Here's a complete example application that will store a sample file for you in Blobstore.
package sample
import (
"bytes"
"net/http"
"mime/multipart"
"appengine"
"appengine/blobstore"
"appengine/urlfetch"
)
const SampleData = `foo,bar,spam,eggs`
func init() {
http.HandleFunc("/test", StoreSomeData)
http.HandleFunc("/upload", Upload)
}
func StoreSomeData(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
// First you need to create the upload URL:
u, err := blobstore.UploadURL(c, "/upload", nil)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
c.Errorf("%s", err)
return
}
// Now you can prepare a form that you will submit to that URL.
var b bytes.Buffer
fw := multipart.NewWriter(&b)
// Do not change the form field, it must be "file"!
// You are free to change the filename though, it will be stored in the BlobInfo.
file, err := fw.CreateFormFile("file", "example.csv")
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
c.Errorf("%s", err)
return
}
if _, err = file.Write([]byte(SampleData)); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
c.Errorf("%s", err)
return
}
// Don't forget to close the multipart writer.
// If you don't close it, your request will be missing the terminating boundary.
fw.Close()
// Now that you have a form, you can submit it to your handler.
req, err := http.NewRequest("POST", u.String(), &b)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
c.Errorf("%s", err)
return
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", fw.FormDataContentType())
// Now submit the request.
client := urlfetch.Client(c)
res, err := client.Do(req)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
c.Errorf("%s", err)
return
}
// Check the response status, it should be whatever you return in the `/upload` handler.
if res.StatusCode != http.StatusCreated {
http.Error(w, err.Error(), http.StatusInternalServerError)
c.Errorf("bad status: %s", res.Status)
return
}
// Everything went fine.
w.WriteHeader(res.StatusCode)
}
func Upload(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
// Here we just checked that the upload went through as expected.
if _, _, err := blobstore.ParseUpload(r); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
c.Errorf("%s", err)
return
}
// Everything seems fine. Signal the other handler using the status code.
w.WriteHeader(http.StatusCreated)
}
Now if you curl http://localhost:8080/test, it will store a file in the Blobstore.
Important: I'm not exactly sure how you would be charged for bandwidth for the request that you make to your own app. At the worst case, you will be charged for internal traffic, which is cheaper from normal bandwidth iirc.