I'm getting errors like this during the reduce step. The MapReduce Overview page reports that all jobs have completed successfully, but half the records that I was trying to write out were not written. What is causing this error, and how can I fix it?:
ApplicationError: 10
Traceback (most recent call last):
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1511, in __call__
rv = self.handle_exception(request, response, e)
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1505, in __call__
rv = self.router.dispatch(request, response)
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1253, in default_dispatcher
return route.handler_adapter(request, response)
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1077, in __call__
return handler.dispatch()
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 547, in dispatch
return self.handle_exception(e, self.app.debug)
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 545, in dispatch
return method(*args, **kwargs)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/base_handler.py", line 65, in post
self.handle()
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/handlers.py", line 181, in handle
entity, input_reader, ctx, tstate)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/handlers.py", line 298, in process_data
output_writer.write(output, ctx)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/output_writers.py", line 693, in write
ctx.get_pool("records_pool").append(str(data))
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/output_writers.py", line 336, in append
self.flush()
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/output_writers.py", line 371, in flush
COUNTER_IO_WRITE_BYTES, len(str_buf))(self._ctx)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/lib/files/file.py", line 297, in __exit__
self.close()
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/lib/files/file.py", line 291, in close
self._make_rpc_call_with_retry('Close', request, response)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/lib/files/file.py", line 427, in _make_rpc_call_with_retry
_make_call(method, request, response)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/lib/files/file.py", line 252, in _make_call
_raise_app_error(e)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/lib/files/file.py", line 195, in _raise_app_error
raise FileNotOpenedError(e)
FileNotOpenedError: ApplicationError: 10
Traceback (most recent call last):
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/main.py", line 105, in main
util.run_wsgi_app(APP)
File "/python27_runtime/python27_lib/versions/1/google/appengine/ext/webapp/util.py", line 98, in run_wsgi_app
run_bare_wsgi_app(add_wsgi_middleware(application))
File "/python27_runtime/python27_lib/versions/1/google/appengine/ext/webapp/util.py", line 116, in run_bare_wsgi_app
result = application(env, _start_response)
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1519, in __call__
response = self._internal_error(e)
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1511, in __call__
rv = self.handle_exception(request, response, e)
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1505, in __call__
rv = self.router.dispatch(request, response)
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1253, in default_dispatcher
return route.handler_adapter(request, response)
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1077, in __call__
return handler.dispatch()
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 547, in dispatch
return self.handle_exception(e, self.app.debug)
File "/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 545, in dispatch
return method(*args, **kwargs)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/base_handler.py", line 65, in post
self.handle()
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/handlers.py", line 181, in handle
entity, input_reader, ctx, tstate)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/handlers.py", line 298, in process_data
output_writer.write(output, ctx)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/output_writers.py", line 693, in write
ctx.get_pool("records_pool").append(str(data))
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/output_writers.py", line 336, in append
self.flush()
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/output_writers.py", line 371, in flush
COUNTER_IO_WRITE_BYTES, len(str_buf))(self._ctx)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/lib/files/file.py", line 297, in __exit__
self.close()
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/lib/files/file.py", line 291, in close
self._make_rpc_call_with_retry('Close', request, response)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/lib/files/file.py", line 427, in _make_rpc_call_with_retry
_make_call(method, request, response)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/lib/files/file.py", line 252, in _make_call
_raise_app_error(e)
File "/base/data/home/apps/s~getaround3/dont-make-this-default-export-mr.366029363091232181/mapreduce/lib/files/file.py", line 195, in _raise_app_error
raise FileNotOpenedError(e)
FileNotOpenedError: ApplicationError: 10
I increased the number of shards which seems to have fixed the problem. Increasing the number of shards reduces the amount of time each slice has to hold a file open which may have been the cause of the problem I had.
Holding the file open for more than 30 seconds will also result in this error. Try to break the writes into multiple writes.
refer link for deatils https://groups.google.com/forum/#!topic/google-appengine-java/5XBVaVDKvvs
Related
I started learning C programming language and I wrote popular "Hello world!" program.
but when I try to execute it, this appears
Traceback (most recent call last):
File "C:\Users\asebo\AppData\Local\Programs\Python\Python39\lib\runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "C:\Users\asebo\AppData\Local\Programs\Python\Python39\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "c:\Users\asebo\.vscode\extensions\ms-python.python-2021.3.680753044\pythonFiles\lib\python\debugpy\__main__.py", line 45, in <module>
cli.main()
File "c:\Users\asebo\.vscode\extensions\ms-python.python-2021.3.680753044\pythonFiles\lib\python\debugpy/..\debugpy\server\cli.py", line 444, in main
run()
File "c:\Users\asebo\.vscode\extensions\ms-python.python-2021.3.680753044\pythonFiles\lib\python\debugpy/..\debugpy\server\cli.py", line 285, in run_file
runpy.run_path(target_as_str, run_name=compat.force_str("__main__"))
File "C:\Users\asebo\AppData\Local\Programs\Python\Python39\lib\runpy.py", line 267, in run_path
code, fname = _get_code_from_file(run_name, path_name)
File "C:\Users\asebo\AppData\Local\Programs\Python\Python39\lib\runpy.py", line 242, in _get_code_from_file
code = compile(f.read(), fname, 'exec')
File "c:\Users\asebo\OneDrive\Počítač\codeblock\c.c", line 2
int main() {
^
SyntaxError: invalid syntax
And here is my full code:
#include <stdio.h>
int main() {
printf("Hello, World!");
return 0;
}
P.S: I am using VS code IDE
Looks like you've got a few Python paths in your compilation output, why?
If I were you, I'd just create a C++ console app. in VS, write it and save it as C code, give it a file name, i.e., "fileName.c" and Debug it from the VS File Menu.
I'm training file handling in C and I am trying to change a single line or position of the file with fseek() while writing and reading with fread() and fwrite() without changing a variable and writing the whole file again, but apparently neither append mode nor write mode let you do this as I've tried with the example below:
void main()
{
FILE *file;
char answer;
char text[7] = "text 1\n";
char text2[7] = "text 2\n";
file = fopen("fseek.txt", "w"); //creating 'source' file
fwrite(text, sizeof(char), sizeof(text), file);
fwrite(text, sizeof(char), sizeof(text), file);
fwrite(text, sizeof(char), sizeof(text), file);
fclose(file);
scanf("%c", &answer);
switch(answer)
{
case 'a':
//attempt to change single line with append mode
file = fopen("fseek.txt", "a");
fseek(file, 7, SEEK_SET); //7 characters offset is the second line of the file
fwrite(text2, sizeof(char), sizeof(text), file);
fclose(file);
break;
case 'w':
//attempt to change single line with write mode
file = fopen("fseek.txt", "w");
fseek(file, 7, SEEK_SET); //7 characters offset is the second line of the file
fwrite(text2, sizeof(char), sizeof(text), file);
fclose(file);
break;
}
}
But with append mode it just writes the variable on the end of the file even with the fseek() function beforehand and write mode just erases the file and rewrites it. So how would I change a single line of a file using fseek() or similars ?
You need to open in r+ mode. w mode empties the file first, r doesn't, since it's for reading the file. The + modifier allows you to write the file as well.
When you change the line, the new text needs to be the same length as the original line. If it's shorter, the remainder of the original line will be left in the file. If it's longer, you'll overwrite the beginning of the next line.
There seems to be an issue in
google-cloud-pubsub==0.39.1
google-api-python-client==1.7.8
In which doing a pull in a loop when the credentials goes bad causes a flood of python3 1456 root 71u IPv4 46501 0t0 TCP XXX-XXXXX-XXXX:47074->YYYYYYYY-YY-YYYYY.1e100.net:https (CLOSE_WAIT)files to remain open and eventually cause a "too many files open" issue.
The problem does not appear to be in pubsub itself but in gRPC.
May 8 22:34:41 .sh[17736]: Traceback (most recent call last):
May 8 22:34:41 .sh[17736]: File "/opt///lib/python3.6/site-packages/google/api_core/grpc_helpers.py", line 57, in error_remapped_callable
May 8 22:34:41 .sh[17736]: return callable_(*args, **kwargs)
May 8 22:34:41 .sh[17736]: File "/opt///lib/python3.6/site-packages/grpc/_channel.py", line 562, in call
May 8 22:34:41 .sh[17736]: return _end_unary_response_blocking(state, call, False, None)
May 8 22:34:41 .sh[17736]: File "/opt///lib/python3.6/site-packages/grpc/_channel.py", line 466, in _end_unary_response_blocking
May 8 22:34:41 .sh[17736]: raise _Rendezvous(state, None, None, deadline)
May 8 22:34:41 .sh[17736]: grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
May 8 22:34:41 .sh[17736]: #011status = StatusCode.UNAVAILABLE
May 8 22:34:41 .sh[17736]: #011details = "Getting metadata from plugin failed with error: ('invalid_grant: Invalid JWT Signature.', '{\n "error": "invalid_grant",\n "error_description": "Invalid JWT Signature."\n}')"
May 8 22:34:41 .sh[17736]: #011debug_error_string = "{"created":"#1557354881.258250528","description":"Getting metadata from plugin failed with error: ('invalid_grant: Invalid JWT Signature.', '{\n "error": "invalid_grant",\n "error_description": "Invalid JWT Signature."\n}')","file":"src/core/lib/security/credentials/plugin/plugin_credentials.cc","file_line":79,"grpc_status":14}"
May 8 22:34:41 .sh[17736]: >
May 8 22:34:41 .sh[17736]: The above exception was the direct cause of the following exception:
May 8 22:34:41 .sh[17736]: Traceback (most recent call last):
May 8 22:34:41 .sh[17736]: File "/opt///lib/python3.6/site-packages/lt/cloud/cloudpull.py", line 113, in subscribeToStuff
May 8 22:34:41 .sh[17736]: pull_response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES, timeout=60, retry=None)
May 8 22:34:41 .sh[17736]: File "/opt///lib/python3.6/site-packages/google/cloud/pubsub_v1/_gapic.py", line 45, in
May 8 22:34:41 .sh[17736]: fx = lambda self, *a, **kw: wrapped_fx(self.api, *a, **kw) # noqa
May 8 22:34:41 .sh[17736]: File "/opt///lib/python3.6/site-packages/google/cloud/pubsub_v1/gapic/subscriber_client.py", line 860, in pull
May 8 22:34:41 .sh[17736]: request, retry=retry, timeout=timeout, metadata=metadata
May 8 22:34:41 .sh[17736]: File "/opt///lib/python3.6/site-packages/google/api_core/gapic_v1/method.py", line 143, in call
May 8 22:34:41 .sh[17736]: return wrapped_func(*args, **kwargs)
May 8 22:34:41 .sh[17736]: File "/opt///lib/python3.6/site-packages/google/api_core/timeout.py", line 102, in func_with_timeout
May 8 22:34:41 .sh[17736]: return func(*args, **kwargs)
May 8 22:34:41 .sh[17736]: File "/opt///lib/python3.6/site-packages/google/api_core/grpc_helpers.py", line 59, in error_remapped_callable
May 8 22:34:41 .sh[17736]: six.raise_from(exceptions.from_grpc_error(exc), exc)
May 8 22:34:41 .sh[17736]: File "", line 3, in raise_from
May 8 22:34:41 .sh[17736]: google.api_core.exceptions.ServiceUnavailable: 503 Getting metadata from plugin failed with error: ('invalid_grant: Invalid JWT Signature.', '{\n "error": "invalid_grant",\n "error_description": "Invalid JWT Signature."\n}')
https://github.com/googleapis/google-cloud-python/issues/5523
Notice at the bottom client.api.transport._channel.close()
A pubsub_v1.SubscriberClient falls into the client category.
So you can do like...
subscriber = pubsub_v1.SubscriberClient(credentials=credentials)
try:
subscription_path = subscriber.subscription_path(project, subscription)
pull_response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES, timeout=60, retry=None)
for msg in pull_response.received_messages:
#do stuff
finally:
subscriber.api.transport._channel.close()
You can do this in a loop and keep your process from opening too many files :)
Didn't change code for a month. But today (2012-11-13) started to get strange error "UnknownError: ApplicationError: 7". NOTE: on local GAE function works without errors.
app code from https://github.com/blueimp/jQuery-File-Upload/blob/master/server/gae-python/main.py
ApplicationError: 7
Traceback (most recent call last):
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1511, in __call__
rv = self.handle_exception(request, response, e)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1505, in __call__
rv = self.router.dispatch(request, response)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1253, in default_dispatcher
return route.handler_adapter(request, response)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1077, in __call__
return handler.dispatch()
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 547, in dispatch
return self.handle_exception(e, self.app.debug)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 545, in dispatch
return method(*args, **kwargs)
File "/base/data/home/apps/s~eventinarea-images/1.362351013166724659/main.py", line 132, in post
s = json.dumps(self.handle_upload(), separators=(',',':'))
File "/base/data/home/apps/s~eventinarea-images/1.362351013166724659/main.py", line 90, in handle_upload
self.write_blob(fieldStorage.value, result)
File "/base/data/home/apps/s~eventinarea-images/1.362351013166724659/main.py", line 74, in write_blob
files.finalize(blob)
File "/base/python27_runtime/python27_lib/versions/1/google/appengine/api/files/file.py", line 568, in finalize
f.close(finalize=True)
File "/base/python27_runtime/python27_lib/versions/1/google/appengine/api/files/file.py", line 291, in close
self._make_rpc_call_with_retry('Close', request, response)
File "/base/python27_runtime/python27_lib/versions/1/google/appengine/api/files/file.py", line 427, in _make_rpc_call_with_retry
_make_call(method, request, response)
File "/base/python27_runtime/python27_lib/versions/1/google/appengine/api/files/file.py", line 252, in _make_call
_raise_app_error(e)
File "/base/python27_runtime/python27_lib/versions/1/google/appengine/api/files/file.py", line 186, in _raise_app_error
raise UnknownError(e)
UnknownError: ApplicationError: 7
I got the same problem but it seems to have been fixed silently
'TransactionOptions' object is not callable
Traceback (most recent call last):
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1511, in __call__
rv = self.handle_exception(request, response, e)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1505, in __call__
rv = self.router.dispatch(request, response)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1253, in default_dispatcher
return route.handler_adapter(request, response)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 1077, in __call__
return handler.dispatch()
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 547, in dispatch
return self.handle_exception(e, self.app.debug)
File "/base/python27_runtime/python27_lib/versions/third_party/webapp2-2.3/webapp2.py", line 545, in dispatch
return method(*args, **kwargs)
File "/base/data/home/apps/.../0-1.361943419475938372/cron.py", line 26, in get
db.run_in_transaction(xg_on, transactional_task)
File "/base/python27_runtime/python27_lib/versions/1/google/appengine/api/datastore.py", line 2433, in RunInTransaction
return RunInTransactionOptions(None, function, *args, **kwargs)
File "/base/python27_runtime/python27_lib/versions/1/google/appengine/api/datastore.py", line 2571, in RunInTransactionOptions
ok, result = _DoOneTry(new_connection, function, args, kwargs)
File "/base/python27_runtime/python27_lib/versions/1/google/appengine/api/datastore.py", line 2593, in _DoOneTry
result = function(*args, **kwargs)
TypeError: 'TransactionOptions' object is not callable
my code in cron.py looks like:
def transactional_task():
doStuff()
xg_on = db.create_transaction_options(xg=True)
db.run_in_transaction(xg_on, transactional_task)
lifted straight from the docs here
any idea why app engine thinks TransactionOptions can't be created? My datastore is High Replication.
Which docs? db.run_in_transaction() takes a first argument that is a function. I think you meant to call run_in_transaction_options() which does take a TransactionOptions objects as its first argument.