Google appengine blobstore upload handler processing extra form post parameters - google-app-engine

I wish to have a file upload form that in addition to the file selection input , also has other input fields like textarea, dropdown etc. The problem is that I cannot access any post parameters other than the file in my blobstore upload handler. I am using the following function call to get the parameter name but it always returns an empty screen.
par = self.request.get("par")
I found another question with a similar problem Uploading a video to google app engine blobstore. The answer to that question suggests a workaround solution to set the filename to the parameter you wish to read which is not sufficient for my needs. Is there a way to access other form parameters in the post method of blobstore upload handler?

Did you find the solution?
In my experience, when using form/multipart request doesn't include the other parameters and they have to be dug out manually.
This is how I dig out parameters out of request that is used to send a file.
import java.util.Map;
import java.util.HashMap;
import java.util.Iterator;
import javax.servlet.http.HttpServletRequest;
// for reading form data when posted with multipart/form-data
import java.io.*;
import javax.servlet.ServletException;
import org.apache.commons.fileupload.FileItemStream;
import org.apache.commons.fileupload.FileItemIterator;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import com.google.appengine.api.datastore.Blob;
// Fetch the attributes for a given model using rails conventions.
// We need to do this in Java because getParameterMap uses generics.
// We currently only support one lever: foo[bar] but not foo[bar][baz].
// We currently only pull the first value, so no support for checkboxes
public class ScopedParameterMap {
public static Map params(HttpServletRequest req, String model)
throws ServletException, IOException {
Map<String, Object> scoped = new HashMap<String, Object>();
if (req.getHeader("Content-Type").startsWith("multipart/form-data")) {
try {
ServletFileUpload upload = new ServletFileUpload();
FileItemIterator iterator = upload.getItemIterator(req); // this is used to get those params
while (iterator.hasNext()) {
FileItemStream item = iterator.next();
InputStream stream = item.openStream();
String attr = item.getFieldName();
if (attr.startsWith(model + "[") && attr.endsWith("]")) { // fetches all stuff like article[...], you can modify this to return only one value
int len = 0;
int offset = 0;
byte[] buffer = new byte[8192];
ByteArrayOutputStream file = new ByteArrayOutputStream();
while ((len = stream.read(buffer, 0, buffer.length)) != -1) {
offset += len;
file.write(buffer, 0, len);
}
String key = attr.split("\\[|\\]")[1];
if (item.isFormField()) {
scoped.put(key, file.toString());
} else {
if (file.size() > 0) {
scoped.put(key, file.toByteArray());
}
}
}
}
} catch (Exception ex) {
throw new ServletException(ex);
}
} else {
Map params = req.getParameterMap();
Iterator i = params.keySet().iterator();
while (i.hasNext()) {
String attr = (String) i.next();
if (attr.startsWith(model + "[") && attr.endsWith("]")) {
String key = attr.split("\\[|\\]")[1];
String val = ((String[]) params.get(attr))[0];
scoped.put(key, val);
// TODO: when multiple values, set a List instead
}
}
}
return scoped;
}
}
I hope this speedy answer helps, let me know if you have questions.

Related

'ControllerBase.File(byte[], string)' is a method, which is not valid in the given context (CS0119) - in method

I am trying to create an app where user can upload a text file, and gets the altered text back.
I am using React as FE and ASP.NET Core for BE and Azure storage for the database storage.
This is how my HomeController looks like.
I created a separate "UploadToBlob" method, to post the data
public class HomeController : Controller
{
private readonly IConfiguration _configuration;
public HomeController(IConfiguration Configuration)
{
_configuration = Configuration;
}
public IActionResult Index()
{
return View();
}
[HttpPost("UploadFiles")]
//OPTION B: Uncomment to set a specified upload file limit
[RequestSizeLimit(40000000)]
public async Task<IActionResult> Post(List<IFormFile> files)
{
var uploadSuccess = false;
string uploadedUri = null;
foreach (var formFile in files)
{
if (formFile.Length <= 0)
{
continue;
}
// read directly from stream for blob upload
using (var stream = formFile.OpenReadStream())
{
// Open the file and upload its data
(uploadSuccess, uploadedUri) = await UploadToBlob(formFile.FileName, null, stream);
}
}
if (uploadSuccess)
{
//return the data to the view, which is react display text component.
return View("DisplayText");
}
else
{
//create an error component to show there was some error while uploading
return View("UploadError");
}
}
private async Task<(bool uploadSuccess, string uploadedUri)> UploadToBlob(string fileName, object p, Stream stream)
{
if (stream is null)
{
try
{
string connectionString = Environment.GetEnvironmentVariable("AZURE_STORAGE_CONNECTION_STRING");
// Create a BlobServiceClient object which will be used to create a container client
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
//Create a unique name for the container
string containerName = "textdata" + Guid.NewGuid().ToString();
// Create the container and return a container client object
BlobContainerClient containerClient = await blobServiceClient.CreateBlobContainerAsync(containerName);
string localPath = "./data/";
string textFileName = "textdata" + Guid.NewGuid().ToString() + ".txt";
string localFilePath = Path.Combine(localPath, textFileName);
// Get a reference to a blob
BlobClient blobClient = containerClient.GetBlobClient(textFileName);
Console.WriteLine("Uploading to Blob storage as blob:\n\t {0}\n", blobClient.Uri);
FileStream uploadFileStream = File.OpenRead(localFilePath);
await blobClient.UploadAsync(uploadFileStream, true);
uploadFileStream.Close();
}
catch (StorageException)
{
return (false, null);
}
finally
{
// Clean up resources, e.g. blob container
//if (blobClient != null)
//{
// await blobClient.DeleteIfExistsAsync();
//}
}
}
else
{
return (false, null);
}
}
}
but the console throws errors, saying "'ControllerBase.File(byte[], string)' is a method, which is not valid in the given context (CS0119)"
And because of this error, another error follows "'HomeController.UploadToBlob(string, object, Stream)': not all code paths return a value (CS0161)"
my questions are
Is it a better idea to create a separate method like I did?
how can I resolve the issue regarding the "File" being valid inside of the UploadToBlob method?
If I want to add the file type validation, where should it happen? t.ex. only text file is alid
If I want to read the text string from the uploaded text file, where should I call the
string contents = blob.DownloadTextAsync().Result;
return contents;
How can I pass down the "contents" to my react component? something like this?
useEffect(() => {
fetch('Home')
.then(response => response.json())
.then(data => {
setForcasts(data)
})
}, [])
Thanks for helping this super newbie with ASP.NET Core!
1) It is ok to put uploading into separate method, it could also be put into a separate class for handling blob operations
2) File is the name of one of the controllers methods, if you want to reference the File class from System.IO namespace, you need to fully qualify the name
FileStream uploadFileStream = System.IO.File.OpenRead(localFilePath);
To the other compile error, you need to return something from the UploadToBlob method, now it does not return anything from the try block
3) File type validation can be put into the controller action method
4) it depends on what you plan to do with the text and how are you going to use it. Would it be a new action of the controller (a new API endpoint)?
5) you could create a new API endpoint for downloading files
UPDATE:
For word replacement you could use a similar method:
private Stream FindMostFrequentWordAndReplaceIt(Stream inputStream)
{
using (var sr = new StreamReader(inputStream, Encoding.UTF8)) // what is the encoding of the text?
{
var allText = sr.ReadToEnd(); // read all text into memory
// TODO: Find most frequent word in allText
// replace the word allText.Replace(oldValue, newValue, stringComparison)
var resultText = allText.Replace(...);
var result = new MemoryStream();
using (var sw = new StreamWriter(result))
{
sw.Write(resultText);
}
result.Position = 0;
return result;
}
}
it would be used in your Post method this way:
using (var stream = formFile.OpenReadStream())
{
var streamWithReplacement = FindMostFrequentWordAndReplaceIt(stream);
// Upload the replaced text:
(uploadSuccess, uploadedUri) = await UploadToBlob(formFile.FileName, null, streamWithReplacement);
}
You probably have this method inside MVC controller in which File method exists. Add in your code System.IO.File instead of File

Spring Cloud Open Feign: Decoder for ByteArrayResource

I have a Spring Boot Rest End Point defined in an interface to download an image
#GetMapping(value = "/{name}")
ResponseEntity<ByteArrayResource> getFileByName(#PathVariable("name") String name);
And I use Feign Builder to invoke this end point.
Feign.builder()
.client(new ApacheHttpClient())
.contract(new SpringMvcContract())
.decoder(new JacksonDecoder())
.encoder(new JacksonEncoder())
.target(clazz, url)
On invoking, I get below error
com.fasterxml.jackson.core.JsonParseException: Unexpected character ('�' (code 65533 / 0xfffd)): expected a valid value (number, String, array, object, 'true', 'false' or 'null')
at [Source: (BufferedReader); line: 1, column: 2]
When I try to invoke the end point directly from Insomnia, it works fine. But fails through Feign Builder. The response content type is image/jpeg
Is there any specific decoder in feign to handle ByteArrayResource? I tried ResponseEntityDecoder, StreamDecoder and JacksonDecoder. None of it works.
On debugging, I see that Jackson ObjectMapper readValue fails. I tried changing the return type from ByteArraySource to byte[], didn't work either.
Any help?
I wrote my own little decoder and the problem was resolved. Below is the decoder
private Decoder byteArrayResourceDecoder() {
Decoder decoder = (response, type) -> {
if (type instanceof Class && ByteArrayResource.class.isAssignableFrom((Class) type)) {
return StreamUtils.copyToByteArray(response.body().asInputStream());
}
return new JacksonDecoder().decode(response, type);
};
return new ResponseEntityDecoder(decoder);
}
Hope this template helps others who has similar issues. Would have expected Feign to have decoder that supports all return types.
Thanks Maz - your solution helped me.
I modified your solution for my needs to read Spring StreamingResponseBody
1.) Create the decoder wrapper that either returns JacksonDecoder (Default) or reads the responsebody into a byte array.
Decoder decoder = (response, type) -> {
Map<String, Collection<String>> headers = response.headers();
Collection<String> contentType = null;
for (String x : headers.keySet()){
if ("content-type".equals(x.toLowerCase())){
contentType = headers.get(x);
}
}
if (contentType == null || contentType.stream().filter(x -> x.contains("application/json")).findFirst().isPresent()) {
return new JacksonDecoder(getMapper()).decode(response, type);
}
InputStream initialStream = response.body().asInputStream();
byte[] buffer = new byte[512];
byte[] result = null;
try(ByteArrayOutputStream out = new ByteArrayOutputStream()) {
try {
int length = 0;
while ((length = initialStream.read(buffer)) != -1) {
out.write(buffer, 0, length);
}
} finally {
out.flush();
}
result = out.toByteArray();
} finally {
initialStream.close();
}
return result;
};
2.) Use the custom decoder with the Feign.Builder
Feign.Builder builder = Feign.builder()
// --
.decoder(decoder)
// --
openfeignfeignspringstreamingresponsebody

How to send encrypted (3DES) data into aqueduct without getting any error?

I m using Aqueduct 3.0. I need to learn How to capture post request in Aqueduct 3.0?
My Request: http://127.0.0.1:8888/login/ziD7v0Ul99vmNWnxJRxZIiTY4zakNoq8GjM+oHROYz/YTHnd3NH1XfHRULY0jaHU
Get a Response:
[INFO] aqueduct: GET /login/ziD7v0Ul99vmNWnxJRxZIiTY4zakNoq8GjM+oHROYz/YTHnd3NH1XfHRULY0jaHU 11ms 404
my channel.dart routing
// TODO: connect to Socket **********
router.route('/login/[:value]').link(() {
return new LoginController();
//..contentType = ContentType.TEXT;
});
my LoginController.dart
import 'package:aqueduct/aqueduct.dart';
import 'package:niyaziapi/niyaziapi.dart';
import 'package:niyaziapi/util/niyaziGetPrivate.dart';
import 'package:niyaziapi/util/niyaziSetPrivate.dart';
class LoginController extends Controller {
String _xCustomerToken;
String _xCustomerName;
String _xPrivate;
String _xResult;
String _xRequestValue;
String _xReply;
#override
Future<RequestOrResponse> processRequest(Request request) async {
String tempData = request.toString();
print("tempDate: $tempData"); // can’t print
try {
if (request.path.variables.containsKey('value')) {
_xPrivate = (request.path.variables['value']).trim();
print("_xPrivate: $_xPrivate");
var decryptedData = await getPrivate(_xPrivate);
var decryptedList = decryptedData.split(":_:");
decryptedData = null;
decryptedData = "Q101:_:" + decryptedList[2].toString() + ":_:" + decryptedList[3].toString();
print(decryptedData);
var socket = await Socket.connect('192.168.1.22', 1024);
socket.write("$decryptedData\r\n");
await for (var data in socket) {
_xReply = new String.fromCharCodes(data).trim();
var list = _xReply.split(":_:");
_xCustomerToken = list[2].toString();
_xCustomerName = list[3].toString();
});
_xResult = "$_xCustomerToken:_:$_xCustomerName";
var encryptedData = await setPrivate(_xResult);
return new Response.ok("$encryptedData");
}
} else {
return new Response.ok("404: Wrong Request");
}
} catch (e) {
return new Response.ok("404: $e.errorMessage");
}
}
}
when I testing I found that my code works. Only reason that I am sending 3DES data and has + and / character in it.
If you look at closely in first request, there is a + and / character in data which give me an error.
/login/ziD7v0Ul99vmNWnxJRxZIiTY4zakNoq8GjM+oHROYz/YTHnd3NH1XfHRULY0jaHU 19ms 404
on the other hand if I remove those character than I get perfect response.
/login/ziD7v0Ul99vmNWnxJRxZIiTY4zakNoq8GjMoHROYzYTHnd3NH1XfHRULY0jaHU 13 ms 200
So, question comes how to send encrypted (3DES) data into aqueduct without getting any error?
Going to Like Aqueduct twice :)
It was very simple:
var _xPrivate = (request.path.variables['value']).trim(); change to:
var _xPrivate = request.path.remainingPath;
print("request: $_xPrivate");

Sending LDAP request with message ID

Need to send an LDAP search request with message ID set to 0 value (as part of RFC validation testing). Tried the following modified code from apache directory api examples section:
import java.io.IOException;
import org.apache.directory.api.ldap.model.entry.DefaultEntry;
import org.apache.directory.api.ldap.model.entry.ModificationOperation;
import org.apache.directory.api.ldap.model.exception.LdapException;
import org.apache.directory.api.ldap.model.name.Dn;
import org.apache.directory.ldap.client.api.LdapConnection;
import org.apache.directory.ldap.client.api.LdapNetworkConnection;
import org.apache.directory.api.ldap.model.message.SearchRequest;
import org.apache.directory.api.ldap.model.message.SearchRequestImpl;
import org.apache.directory.api.ldap.model.cursor.SearchCursor;
import org.apache.directory.api.ldap.model.exception.LdapNoPermissionException;
import org.apache.directory.api.ldap.model.name.Dn;
import org.apache.directory.api.ldap.model.exception.LdapInvalidDnException;
public class ManageLDAPConnection {
private static Dn getSafeSearchBaseDn(String dn) throws LdapInvalidDnException{
Dn searchBaseDn = null;
if (dn != null && !dn.isEmpty()){
searchBaseDn = new Dn(dn);
}else{
searchBaseDn = Dn.ROOT_DSE;
}
return searchBaseDn;
}
public static void main (String[] args) {
int messageId = 0;
int port = 389;
String username = "<Admin CN>";
String password = "<Password>";
String hostname = "<IP>";
SearchCursor searchResult = null;
String dn = "<DN>";
String filterExpr = "(objectclass=*)";
org.apache.directory.api.ldap.model.message.SearchScope searchScopeValue = org.apache.directory.api.ldap.model.message.SearchScope.OBJECT;
LdapConnection connection = new LdapNetworkConnection(hostname, port);
try {
connection.bind(username, password);
System.out.println("Connected successfully");
} catch (LdapException e) {
System.out.println("Unable to bind");
}
try {
SearchRequest searchRequest = new SearchRequestImpl();
System.out.println(searchRequest.getMessageId());
searchRequest.setMessageId(0);
System.out.println(searchRequest.getMessageId());
searchRequest.setBase(getSafeSearchBaseDn(dn));
searchRequest.setFilter(filterExpr);
searchRequest.setScope(searchScopeValue);
searchResult = connection.search(searchRequest);
} catch (LdapNoPermissionException e){
System.out.println("No permission exception");
} catch (LdapException e){
System.out.println("LDAP Exception: " + e.getMessage());
}
}
}
The above code is able to send the request, but the message ID is still sent as non zero,
even though the following has been done:
searchRequest.setMessageId(0);
You're clearly going to have to use a different library, or modify this one, or go to a lower level. It isn't at all surprising that this library prevents you from shooting yourself in the foot.
Had some solution in python's pyasn1-modules. The following seems to work well:
from pyasn1.type import univ, namedval, namedtype, tag
from pyasn1.codec.ber import encoder
import socket
from pyasn1_modules.rfc2251 import *
ldap_bind_request = BindRequest()
ldap_bind_request.setComponentByName('version', 3)
ldap_bind_request.setComponentByName('name', 'cn=admin,o=org')
ldap_auth = AuthenticationChoice()
ldap_auth.setComponentByName('simple', 'mypwd')
ldap_bind_request.setComponentByName('authentication', ldap_auth)
ldap_message = LDAPMessage()
ldap_message.setComponentByName('messageID', 0)
ldap_message.setComponentByName('protocolOp', ldap_bind_request)
print(ldap_bind_request.prettyPrint())
print(dir(ldap_bind_request))
encoded_request = encoder.encode(ldap_message)
print(encoded_request)
asock = socket.socket()
asock.connect(('127.0.0.1', 389))
asock.send(encoded_request)
There is something named JAVA ASN.1 Compiler (JAC). Trying to see if they provide something similar, with less of object oriented complexity which is common in java :)

How can I access the number of errors that my App Engine app is returning per hour/day/week?

Is there an API that my Google App Engine app can call when signed in as the as the app admin that would return information on the number of errors (404, 500, etc.) my app is returning?
I'd like to setup a simple cron job in my application to count the number of errors that my app is returning every few minutes and send me an email if the error rate becomes unexpectedly high. I'd like to avoid having to scrape the information from the Appspot dashboard or run another process outside of my app.
The closest thing to what you need is probably LogService API
Note that it isn't available for Java runtime (yet, I assume).
This only works on the cloud and is GAE-friendly.
You will need jsoup.
package some.package
import java.util.HashMap;
import java.util.Map;
import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class AppEngineScraperUtil
{
private static final Logger logger = LoggerFactory.getLogger( AppEngineScraperUtil.class );
/**
* #param appId
* in the form of {#code appId}
* #return dashboard in html
*/
public static String fetchDashboard( String appId )
{
return jsoupWay( appId );
}
private static String jsoupWay( String appId )
{
try
{
Connection conn = createGetConn( "https://appengine.google.com" );
Response result = conn.execute();
Document doc = result.parse();
// parse inputs
Elements elements = doc.select( "#dsh, [name=GALX], #service, #continue, #ltmpl" );
Map<String, String> formFields = new HashMap<String, String>();
// build form
for ( Element element: elements )
formFields.put( element.attr( "name" ), element.val() );
formFields.put( "Email", "xxx" );
formFields.put( "Passwd", "xxx" );
String formAction = doc.select( "form" ).first().attr( "action" );
// parse cookies
Map<String, String> cookies = result.cookies();
// build post
conn = createPostConn( formAction );
conn.cookies( cookies );
conn.data( formFields );
conn.header( "Content-Type", "application/x-www-form-urlencoded" );
result = conn.execute();
doc = result.parse();
// get dashboard
conn = createGetConn( "https://appengine.google.com/dashboard?&app_id=" + appId );
conn.cookies( result.cookies() );
result = conn.execute();
// return html
doc = result.parse();
return doc.toString();
}
catch ( Exception e )
{
logger.error( "Error retrieving dashboard.", e );
}
return null;
}
private static Connection createPostConn( String url )
{
Connection conn = Jsoup.connect( url );
conn.method( Method.POST );
return conn;
}
private static Connection createGetConn( String url )
{
Connection conn = Jsoup.connect( url );
conn.method( Method.GET );
return conn;
}
}

Resources