I'm trying to send a list from server to client.
The list looks like this (it's a csv file).
201,8,0040000080
205,8,1f421d25721e
but when sending I get this error:
TypeError: must be string or buffer, not list
I tried 2 options:
Iterate through the list and send each string to the server, but got this as a result:
201 ---> 2,0,1
Tried casting each line, e.g str(line), and then send it, but got this:
201,8,0040000080 ---> [,',2,0,1,',",", ,',8,',",", ,',0,0,4,0,0,0,0,0,8,0,',]
how can I solve this? I just want to send the data from the client to server as is. For the record, the Client code:
import socket
import csv
clientSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
with open('can_data.csv', 'rb') as csv_file:
csv_reader = csv.reader(csv_file)
for line in csv_reader:
clientSock.sendto(str(line), (self.address, self.port))
Server code:
with open('output.csv', 'wb') as new_file:
csv_writer = csv.writer(new_file)
while True:
data, addr = s.recvfrom(1024)
csv_writer.writerow(data)
Both sides need to agree on a serialization format. str(line) may work when paired with ast.literaleval() but repr(line) would be the better choice as repr tries for a more precise representation that str. You could also move to a serialization protocol like pickle or json.
Assuming this is python 3, I also moved the csv to text and used utf-8 encoding on the wire.
client:
import socket
import csv
address = 'localhost'
port = 5555
clientSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
with open('can_data.csv', 'r') as csv_file:
csv_reader = csv.reader(csv_file)
for line in csv_reader:
clientSock.sendto(repr(line).encode('utf-8'), (address, port))
server:
import socket
import csv
import ast
address = 'localhost'
port = 5555
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind((address, port))
with open('output.csv', 'w') as new_file:
csv_writer = csv.writer(new_file)
while True:
data, addr = sock.recvfrom(1024)
csv_writer.writerow(ast.literal_eval(data.decode('utf-8')))
Related
I have managed to get a live raw bytearray file containing just the data. I tried converting it into wave file using the following code:
import scipy.io.wavfile
import numpy as np
import soundfile as sf
def convert_bytes_to_audio():
fi = open("receive_data.txt","r")
for line in fi:
scipy.io.wavfile.write('format.wav', 48000, np.frombuffer(str.encode(line),dtype = 'B'))
fi.close()
def main():
convert_bytes_to_audio()
if __name__ == '__main__':
convert_bytes_to_audio()
The bytearray:
b'\x00Z\x15\x02\xf9\x15q\x9d1$\xd2\xde:\xb44\xaf\x8d_4\xbe\xe5\x02\x1f\x87\xe0,d\xa8YS\x02\xea\xdc\xf8\x85\xcc\x87p\x8be)Fi4FE\xeb\xf5\x10\xfbr\xd3-)\x02J\xd9\xebOo\xca\xd3\x04\xcaaG\xf2\x17\xe6\x04\xd3\xdds(R\x98\xf8\xe6\x80#b\xa3\x0e~h\xb1\x02\xf7"(%\x12y\xb6\x1e/wL\n\x9a\xb39\xd7\xb2&Mc\xca\xdc\x9f\xbb\xff\xa7\x81n\xe213\x85\xea\xc8p\x1c{\x19j\x1a\xdd\x0f\xdd\x9d\xde\x19s\x89]\xf8z\x11R\xed5\xd3g\xacy\x85\xa8\x8f\xca\xad\x91\xba\xbe\x07\xa5\xfa\xa0'
b"\x01Z\x8c\xe3\xd9\xd82N8F\x1a\x9d\r\xdct\x97+\xdf\x7f8\x1dL\xb6\xae\xdb\xc2\xbc\xd4\x8b\x1f\xe7rZ\rI\x85\x7f\xb6F\x85\x15\xc6$\xea\x82&l\r8\x82Zu\x00#\x0b\x81\xe6ij\xeb\x02\xbf[\xb6\xfa\x83\x111\x86\xd0\xf2\xb6\x0f\x94\xedk\x81\x0c\x1b%\x16\xd9\xb31\x0c\x04:\x8c[O\xde'\xae[\xe0\x90W/\x9c\xea\xb4\xcf(}\x9e\x19\xa3\xdc\x16?\xc2\x01L[\x1d\x9c\xcc\x1bqP\x95g\xd2s\xbdNM\xe8p\x9c:rW\x01\xf8\xc9k\xf8V;\x9f\xe3\xb7\xe4\x9c\xd5f27\xe8Kg\xbb\xdf\xb0\xbc2\xf9\x0e<G$*\x91\xbd\xdb\x06\xe9\x9fd\x00\xe5\x8c\x88\xf8R\x93\x91\xab\x84\xc7\xa2\x93\x91\x8c#\x9e\xfd\x1a\xf9FM\xd2\xe9\\\x80\xa5\x18\xa5\xec\xb1\xf8u[\x1a\xe0\xd6\xfda\x07r)\xa9\xf4l\xab\xcf\x15'\xd5\xd41\n\x0c]\x045\x03\r\xde\xc5\xbb;74)\xbe;\x06\x06\xe4{\xbf\x8b\xf1O\xfb\xea7\xbf\x16#(!w\xa8`,\x7f\xc0"
b'\x02Z\x88\xe0f+.\x18 8MG\xa2~_Z\xf4-\x11\x03\xf2w\xf5\x8d\x1d}sCD\xf0\xe9\x97\xf8\x8bg\xe1.0\xf9\x05~\xa4\x17\xf6\x92"\xa7l|\xc8,\xe8>\x14\xca\xb4\x01\x14\x82\xe8,\xcb|\nS\xe8$\xf9\xb5A \xf7\xa9\x9co\n\x19\xda\xcf\xa0\x8c\x97\x9av\xf4\xeb\xd0uw\xf5\xbcQj\xf5\x1b\xca\x0eA\x85\x82\xa60\x81\xcb\x03k\xcf\xa1.#\xee\xa53R\xd0\x87F\x16\xa8\xf3\xa7I\xc4|sQ\x15T\x08R\xdd\x9c\x02\\Y\xb9u\xcb\x80\xea\xbc\xfd\x18O6\xce#\x00\xd5\x15\xa5\xab\r\xf0\x8e6)O\xb8\xd8\xac\xf4\x87x\xdc\x08)\xfc\x9c\xff\xbd\xd0|\xb5kg#\xaa\xa5Gc\xfd\x1c\x1c\xa1\xc7Zl\xb5\xc8\xb3\x07\xa4\xf1]\xa2\xc5\xfb\xd9{]\xffG0\xcc&\xcf\xc8\xbfB*gG\xff\x1fI\x14\x1bs\xa6][\xcea\x11\x7f\x8d\x11\xf9\xdc\xa1\xfcQ\xce\xca\x14\x8d\\\xfd\x08\x9e\x1bD\xa7\x84\x9b5\xb5/\xac->~\x94\x1c\xd2\x83\x08_\\[\x07P\xbf\xc4\xea '
b'\x03Z\x8d\xea\xc5(\xcf\x1e\xa1\x07\x14\xf2\xd7\xe8\x1e\x82d\xe9I\x8fN,(\xd5]\xc4\x83\x14\xe9+\xfb\x93\xfb\xc1\x9e\xf4\xa9l\x95\xbf\x88\x88\xc7\xac\xdd[\xce6\x0bG+gq\x17n[\xb6\x17M\x12\r\x16!"{7M\xd3\xda,4\x9fE\x90\xcd\xae*\xdcf\x14?\xd2=\x00\xf5\x0f+*\xb8Sc|\x00\x86\xed7\xc3\x8e)\xe7\x0c\xe4\xfcml0\x9d\xbd\x03\x90)\xb1\xb2t>S=\xc7\xd3\xe16\xbb\x8f\xbf\xaf\x1c\x8bp>\xa1\xf9\x91\x90GH\xd5|^\x1f\x1d?\x95\xd2\x84\xe4\xc8\xc6\xf3\xc8Si\x05\x95\xbec\xd8{3O\x11M\xb1\x1fL<\x1cR\xdaI\xdc3Hh(\x0e\xd9\xf6\x82\x05\xd3\x9c\x9d\x16\x1e\xf6\'`\xee\xc6\xf2\x1e\xcb\xa1]\xcd\x0b\xff\xaaT\xd7\xbf\xed\x922$\xac\x1aY\xd8ue`\x9c\x0e\x9a\x8b0U\xe42\x92\x00\xe8d\tt\xd7X\x1a]\x9e\xc1\xc5+N\xea\xe3\xc6s\xfd\x86\x8c\x12\x80[\xc9\xaap\x99\x99w\xccX>\xcf[\x1e\xad\x1f0\x03\xe2\xc0'
b"\x04Zo\xc4\x9c\xceL]=S\xd7\xce\xd9cd\t\xe1\n\x01J\xd5\xee\xe93R\xe6\xf2\xbd\x89u P\xafX\xdaDH\x13$\xff^\x1bK\xa6\xb3\x90\xeeq\x94}\x8a-\xf3c\x7f\x7f\xeaF\x06#\x9a\xf7\xd2a\x17\x99\xf5\xf1\x92\xa3\x88\x06\x9a\xe5qn\x82\xd4R\xa3C\xc6\x08Y\x07b_\xf2\x1a\xf1\xe3}r\xde6\xebH\xf4\x1c\x08\xfc\xeaT/A\xb9\x98f\xee\x0b\x95%c\x9e\xe0i\xe0Q\xc5\xff4g\x01\xcf\xc4\xe0c\xc3r\xb1\xf7\xab\x8e5I\t/C\xb3=a\xca\xd0\xb1\xebh#\x88m\xcdt(\x99\xa0\xc6\xb9\xb3\xb5\xee\xf8\x1f\xd3\xe2\x9c\xad'D\xb1\xd5_[\xb0\xe9\xb2\x8dm\xa3*\xae5\xbfB\xf7\xf9\xb5\xd8\x01u\xbdg\xdd\xa2\xc1\xf0\xc2\x9a\x89P\xb0\x87\x0e\xd6\x9a\xf5\xcb\xfc\xe8\x05QKe4\x0f\x8c\x8eH\x9e\xf7\xf6\xbeF\xc7(\x9d\x8a\xe4O#\x91\xdd\xc2\xa9\x8e\xbfv\xf0D\xb4\x8b\xa0"
b'\x05Zv\xe98m|Y\xd0^\x19\xdf#s\xb0\xd3\x11\xb7a]4\xaf\xff\x16K\xea\xea\xbf\xc7b8\xc16\xae\xe3\x81+\x0c]\x81\xb5c4Z\xed\xd9]\xdaPM\x15/\'5rZ\xb5D\xd9b\xe0dOuy\xdc\xb7\xd6^U.<\xa2\xf3\xfbdqH\x0f\xd3\x92O\x98\xd3\xcc\xd1\x1eb\xe4\x9f_V\x08\xfa\xf7\xa9#\xb6h[g\x8a\x08\xd2\x89&\x96b\xfd\xca\x06W\'7e\xbdG\x16\xac\x81X\xea<\xbc\xe8\xf8\xf3\xcd\xbf\x9a\x84M\x85w\xc9\x9fG`\xa4\xa5A?(\x0c,\xf2"\xeb\xc8\xe8\xcd\xc6\xcf\xdb\xc7HDh<\xd1\x0c`e\x90v\xe4\xe1$F\x1a\xf4\x0e\xc83~0\xc6q[\xe0#\xf82#\x0f\x94\xfc+8\x9a\x0f\xaa\xdb\xcf\xda\x85\x90T\xea\x9f\xff \xb6\x16\rZ\x92\xd0dY\x10\xa8\xed\xe7#F\xdc[k\x98\x9c\x8a\x11\xd6Z\x84\xbe\xef\xaf"\x08i\x9f\xf7\x94\x85\x07\xce\xa8.C\xaf\xac(\xc2\xae\x97quC\x95\xbb;\xdb\xbb\xa0'
b'\x06Z\x89\xef\xc9\x7f\x08\xc9\x0b\xac\xbd\x9a"?z\xb0\x002\xfb\xac\x8e\xeb\x89\x1b,\x89Ee>\x82\xad&\x8ft\xf8\x13y\x94\xcd\xd3\xe9\xa0\n\x16\xc0\xcc\x90\xc91%\xd4\xee\xd8\xcd_\xc9\x02\xd21\xf7\x13\xd4\xf7\xba]\x7f^\xcbuZ\x84\xdf%\xa3\x81\x1dw\xaf <^\x94Ng.\xf7\xae\x0f\xec\xbb\xc3\xb6\xbd\xf4\x0c\xea#7=\xba\x08>FZ\xa8%<\xce\xb8S\x8c\xa2\xee_\xc83}\xbb\xc5\x9d\x9f\x15\x14\xe0\xb8\xf9*0\xb8\x87\xca\xa1\x11Y\xde\x8b\xb5\x8e\x16\xb8\xc4rL\xc8&ut\x8e\x19\xf39)F\xcfR)\xe2\xa8\xff\x8b\xb4s\xc5}\xed%\x86\x13\xbc\x03\x12\xdcb\xa6\\E5\x98\x03\xe5%\x9f4\xb0\xac-\x8e\xac\x83\x17\x08\xb5\x0f\x18\xc3\x0e\xbf\xe5S\xa0\xeb9ni\xc9\xd6\r\xdd\x10\xcc&\'\xf5\xdbz\xcf\x1ac\xf0\xdb\xae\xe6D\x9f\x8cq\x14\xa0\xa7\xcfq!M\xc9\xe46\x19"\xbc=\x82\x18\x87\'\xe0>'
I managed to get the wav file but it is unclear and not exactly for the amount of time I recorded it.
Also the format is u8 type, and encoding is pcm_f32le. But I am unsure how to change those.
Generated wav file:
Can anyone help me on this.
I am currently working on Softbanks' robot Pepper and I try to use Watson speech-to-text solution on Pepper's audio buffers remote streaming by using websocket protocol.
I used the answer to that former question NAO robot remote audio problems to find a way to access remotly pepper's audio buffers and that project https://github.com/ibm-dev/watson-streaming-stt to learn how to use websocket protocole to use watson streaming stt.
However, after I open my websocket application, I start sending buffers to watson and after a few sendings, I receive error: 'Unable to transcode from audio/l16;rate=48000;channel=1 to one of: audio/x-float-array; rate=16000; channels=1'
Each time I'm trying to send Pepper's audio buffer to watson, it is unable to understand it.
I compared data I send with data sent in watson streaming stt example (using pyaudio streaming from microphone instead of Pepper's buffer streaming) and I don't see any difference. Both time I'm pretty sure that I am sending a string containing raw chunks of bytes. Which is what Watson asks for in it documentation.
I try to send chunks of 8192 bytes with a sample rate of 48kHz and I can easily convert Pepper's audio buffer in hexa so I don't understand why Watson can't transcode it.
Here is my code:
# -*- coding: utf-8 -*-
#!/usr/bin/env python
import argparse
import base64
import configparser
import json
import threading
import time
from optparse import OptionParser
import naoqi
import numpy as np
import sys
from threading import Thread
import ssl
import websocket
from websocket._abnf import ABNF
CHANNELS = 1
NAO_IP = "172.20.10.12"
class SoundReceiverModule(naoqi.ALModule):
"""
Use this object to get call back from the ALMemory of the naoqi world.
Your callback needs to be a method with two parameter (variable name, value).
"""
def __init__( self, strModuleName, strNaoIp):
try:
naoqi.ALModule.__init__(self, strModuleName );
self.BIND_PYTHON( self.getName(),"callback" );
self.strNaoIp = strNaoIp;
self.outfile = None;
self.aOutfile = [None]*(4-1); # ASSUME max nbr channels = 4
self.FINALS = []
self.RECORD_SECONDS = 20
self.ws_open = False
self.ws_listening = ""
# init data for websocket interfaces
self.headers = {}
self.userpass = "" #userpass and password
self.headers["Authorization"] = "Basic " + base64.b64encode(
self.userpass.encode()).decode()
self.url = ("wss://stream.watsonplatform.net//speech-to-text/api/v1/recognize"
"?model=fr-FR_BroadbandModel")
except BaseException, err:
print( "ERR: abcdk.naoqitools.SoundReceiverModule: loading error: %s" % str(err) );
# __init__ - end
def __del__( self ):
print( "INF: abcdk.SoundReceiverModule.__del__: cleaning everything" );
self.stop();
def start( self ):
audio = naoqi.ALProxy( "ALAudioDevice", self.strNaoIp, 9559 );
self.nNbrChannelFlag = 3; # ALL_Channels: 0, AL::LEFTCHANNEL: 1, AL::RIGHTCHANNEL: 2; AL::FRONTCHANNEL: 3 or AL::REARCHANNEL: 4.
self.nDeinterleave = 0;
self.nSampleRate = 48000;
audio.setClientPreferences( self.getName(), self.nSampleRate, self.nNbrChannelFlag, self.nDeinterleave ); # setting same as default generate a bug !?!
audio.subscribe( self.getName() );
#openning websocket app
self._ws = websocket.WebSocketApp(self.url,
header=self.headers,
on_open = self.on_open,
on_message=self.on_message,
on_error=self.on_error,
on_close=self.on_close)
sslopt={"cert_reqs": ssl.CERT_NONE}
threading.Thread(target=self._ws.run_forever, kwargs = {'sslopt':sslopt}).start()
print( "INF: SoundReceiver: started!" );
def stop( self ):
print( "INF: SoundReceiver: stopping..." );
audio = naoqi.ALProxy( "ALAudioDevice", self.strNaoIp, 9559 );
audio.unsubscribe( self.getName() );
print( "INF: SoundReceiver: stopped!" );
print "INF: WebSocket: closing..."
data = {"action": "stop"}
self._ws.send(json.dumps(data).encode('utf8'))
# ... which we need to wait for before we shutdown the websocket
time.sleep(1)
self._ws.close()
print "INF: WebSocket: closed"
if( self.outfile != None ):
self.outfile.close();
def processRemote( self, nbOfChannels, nbrOfSamplesByChannel, aTimeStamp, buffer ):
"""
This is THE method that receives all the sound buffers from the "ALAudioDevice" module"""
print "receiving buffer"
# self.data_to_send = self.data_to_send + buffer
# print len(self.data_to_send)
#self.data_to_send = ''.join( [ "%02X " % ord( x ) for x in buffer ] ).strip()
self.data_to_send = buffer
#print("buffer type :", type(data))
#print("buffer :", buffer)
#~ print( "process!" );
print( "processRemote: %s, %s, %s, lendata: %s, data0: %s (0x%x), data1: %s (0x%x)" % (nbOfChannels, nbrOfSamplesByChannel, aTimeStamp, len(buffer), buffer[0],ord(buffer[0]),buffer[1],ord(buffer[1])) );
if self.ws_open == True and self.ws_listening == True:
print "sending data"
self._ws.send(self.data_to_send, ABNF.OPCODE_BINARY)
print "data sent"
#print self.data_to_send
aSoundDataInterlaced = np.fromstring( str(buffer), dtype=np.int16 );
#
aSoundData = np.reshape( aSoundDataInterlaced, (nbOfChannels, nbrOfSamplesByChannel), 'F' );
# print "processRemote over"
# processRemote - end
def on_message(self, ws, msg):
print("message")
data = json.loads(msg)
print data
if "state" in data:
if data["state"] == "listening":
self.ws_listening = True
if "results" in data:
if data["results"][0]["final"]:
self.FINALS.append(data)
# This prints out the current fragment that we are working on
print(data['results'][0]['alternatives'][0]['transcript'])
def on_error(self, ws, error):
"""Print any errors."""
print(error)
def on_close(self, ws):
"""Upon close, print the complete and final transcript."""
transcript = "".join([x['results'][0]['alternatives'][0]['transcript']
for x in self.FINALS])
print("transcript :", transcript)
self.ws_open = False
def on_open(self, ws):
"""Triggered as soon a we have an active connection."""
# args = self._ws.args
print "INF: WebSocket: opening"
data = {
"action": "start",
# this means we get to send it straight raw sampling
"content-type": "audio/l16;rate=%d;channel=1" % self.nSampleRate,
"continuous": True,
"interim_results": True,
# "inactivity_timeout": 5, # in order to use this effectively
# you need other tests to handle what happens if the socket is
# closed by the server.
"word_confidence": True,
"timestamps": True,
"max_alternatives": 3
}
# Send the initial control message which sets expectations for the
# binary stream that follows:
self._ws.send(json.dumps(data).encode('utf8'))
# Spin off a dedicated thread where we are going to read and
# stream out audio.
print "INF: WebSocket: opened"
self.ws_open = True
def version( self ):
return "0.6";
def main():
"""initialisation
"""
parser = OptionParser()
parser.add_option("--pip",
help="Parent broker port. The IP address or your robot",
dest="pip")
parser.add_option("--pport",
help="Parent broker port. The port NAOqi is listening to",
dest="pport",
type="int")
parser.set_defaults(
pip=NAO_IP,
pport=9559)
(opts, args_) = parser.parse_args()
pip = opts.pip
pport = opts.pport
# We need this broker to be able to construct
# NAOqi modules and subscribe to other modules
# The broker must stay alive until the program exists
myBroker = naoqi.ALBroker("myBroker",
"0.0.0.0", # listen to anyone
0, # find a free port and use it
pip, # parent broker IP
pport) # parent broker port
"""fin initialisation
"""
global SoundReceiver
SoundReceiver = SoundReceiverModule("SoundReceiver", pip) #thread1
SoundReceiver.start()
try:
while True:
time.sleep(1)
print "hello"
except KeyboardInterrupt:
print "Interrupted by user, shutting down"
myBroker.shutdown()
SoundReceiver.stop()
sys.exit(0)
if __name__ == "__main__":
main()
I would be thankful if anyone had any idea on how to bypass that error or on what to try to get useful info. I first believed that I was sending "wrong" data to watson however after lots of attempts I have no clue on how to fix that problem.
Thank you a lot,
Alex
I am trying to copy over S3 large dataset (larger than RAM) using SQL alchemy.
My constraints are:
I need to use sqlalchemy
I need to keep memory pressure at lowest
I don't want to use the local filsystem as intermediary step to send data to s3
I just want to pipe data from a DB to S3 in a memory efficient way.
I can to do it normal with data sets (using below logic) but with larger dataset I hit a buffer issue.
The first problem I solved is that executing a query usually buffers the result in memory. I use the fetchmany() method.
engine = sqlalchemy.create_engine(db_url)
engine.execution_options(stream_results=True)
results=engine.execute('SELECT * FROM tableX;')
while True:
chunk = result.fetchmany(10000)
if not chunk:
break
On the other side, I have a StringIO buffer that I feed with the fetchmany data check. Then I send its content to s3.
from io import StringIO
import boto3
import csv
s3_resource = boto3.resource('s3')
csv_buffer = StringIO()
csv_writer = csv.writer(csv_buffer, delimiter=';')
csv_writer.writerows(chunk)
s3_resource.Object(bucket, s3_key).put(Body=csv_buffer.getvalue())
The problem I have is essentially a design issue, how do I make these parts work together. Is it even possible in the same runtime?
engine = sqlalchemy.create_engine(db_url)
s3_resource = boto3.resource('s3')
csv_buffer = StringIO()
csv_writer = csv.writer(csv_buffer, delimiter=';')
engine.execution_options(stream_results=True)
results=engine.execute('SELECT * FROM tableX;')
while True:
chunk = result.fetchmany(10000)
csv_writer = csv.writer(csv_buffer, delimiter=';')
csv_writer.writerows(chunk)
s3_resource.Object(bucket, s3_key).put(Body=csv_buffer.getvalue())
if not chunk:
break
I can make it work for one cycle of fetchmany, but not several. Any idea?
I'm assuming that by "make these parts work together" you mean you want a single file in S3 instead of just parts? All you need to do is to create a file object that, when read, will issue a query for the next batch and buffer that. We can make use of python's generators:
def _generate_chunks(engine):
with engine.begin() as conn:
conn = conn.execution_options(stream_results=True)
results = conn.execute("")
while True:
chunk = results.fetchmany(10000)
if not chunk:
break
csv_buffer = StringIO()
csv_writer = csv.writer(csv_buffer, delimiter=';')
csv_writer.writerows(chunk)
yield csv_buffer.getvalue().encode("utf-8")
This is a stream of chunks of your file, so all we need to do is to stitch these together (lazily, of course) into a file object:
class CombinedFile(io.RawIOBase):
def __init__(self, strings):
self._buffer = ""
self._strings = iter(strings)
def read(self, size=-1):
if size < 0:
return self.readall()
if not self._buffer:
try:
self._buffer = next(self._strings)
except StopIteration:
pass
if len(self._buffer) > size:
ret, self._buffer = self._buffer[:size], self._buffer[size:]
else:
ret, self._buffer = self._buffer, b""
return ret
chunks = _generate_chunks(engine)
file = CombinedFile(chunks)
upload_file_object_to_s3(file)
Streaming the file object to S3 is left as an exercise for the reader. (You can probably use put_object.)
I'm trying to send an array over TCP from a server-like script to a client-like one. The array is variable, so the data is sent using packets and then joined together at the client.
The data I'm trying to send is from the MNIST hand-written digits dataset for Deep Learning. The server-side code is:
tcp = '127.0.0.1'
port = 1234
buffer_size = 4096
(X_train, y_train), (X_test, y_test) = mnist.load_data()
test_data = (X_test, y_test)
# Client-side Deep Learning stuff
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((tcp, port))
x = pickle.dumps(test_data)
s.sendall(x)
s.close()
The client-side script loads a Neural Network that uses the test data to predict classes. The script for listening to said data is:
tcp = '127.0.0.1'
port = 1234
buffer_size = 4096
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((tcp, port))
print ('Listening...')
s.listen(1)
conn, addr = s.accept()
data_arr = []
while True:
data_pack = conn.recv(buffer_size)
if not data: break
data_pack += data
my_pickle = b"".join(data_pack)
test_data = pickle.loads(my_pickle)
print ("Received: " + test_data)
conn.close()
# Irrelevant Deep Learning stuff...
The server sends the data without a hitch, but the client crashes when trying to join the packets received by the client (my_pickle = ...) with the following error:
TypeError: sequence item 0: expected a bytes-like object, int found
How should I format the join in order to recreate the data sent and use it for the rest of the script?
I ended up using both Pickle and ZeroMQ to handle the comunication protocol. An advantage of this method is that I can send more than one data package.
On the client side:
ip = '127.0.0.1'
port = '1234'
# ZeroMQ context
context = zmq.Context()
# Setting up protocol (client)
sock = context.socket(zmq.REQ)
sock.bind('tcp://'+ip+':'+port)
print('Waiting for connection at tcp://'+ip+':'+port+'...')
sock.send(pickle.dumps(X_send))
X_answer = sock.recv()
sock.send(pickle.dumps(y_send))
print('Data sent. Waiting for classification...')
y_answer = sock.recv()
print('Done.')
And on the server side:
# ZeroMQ Context
context = zmq.Context()
# Setting up protocol (server)
sock = context.socket(zmq.REP)
ip = '127.0.0.1'
port = '1234'
sock.connect('tcp://'+ip+':'+port)
print('Listening to tcp://'+ip+':'+port+'...')
X_message = sock.recv()
X_test = pickle.loads(X_message)
sock.send(pickle.dumps(X_message))
y_message = sock.recv()
y_test = pickle.loads(y_message)
print('Data received. Starting classification...')
# Classification process
sock.send(pickle.dumps(y_message))
print('Done.')
I am running a Ryu controller and a Mininet instance with 2 hosts and 1 switch like below.
H1---S---H2
Code in Ryu controller
from ryu.base import app_manager
from ryu.controller import ofp_event
from ryu.controller.handler import CONFIG_DISPATCHER, MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_3
from ryu.lib.packet import packet
from ryu.lib.packet import ethernet
from ryu.lib.packet import ether_types
class SimpleSwitch13(app_manager.RyuApp):
OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION]
def __init__(self, *args, **kwargs):
super(SimpleSwitch13, self).__init__(*args, **kwargs)
self.mac_to_port = {}
#set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER)
def switch_features_handler(self, ev):
datapath = ev.msg.datapath
ofproto = datapath.ofproto
parser = datapath.ofproto_parser
Basically the switch flow table is empty. In this case, when I run h1 ping h2 from my mininet console and record the packet exchanges, this is what I get in wireshark from host h1.
There is no router in the mininet instance. How am I receiving an ICMP Host Destination Unreachable Message from the same host that initiated the ping?
The app code you posted is not complete.
For complete simple_switch_13.py, you can get it from the osrg github.
Take a look, it is like this:
class SimpleSwitch13(app_manager.RyuApp):
OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION]
def __init__(self, *args, **kwargs):
super(SimpleSwitch13, self).__init__(*args, **kwargs)
self.mac_to_port = {}
#set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER)
def switch_features_handler(self, ev):
datapath = ev.msg.datapath
ofproto = datapath.ofproto
parser = datapath.ofproto_parser
match = parser.OFPMatch()
actions = [parser.OFPActionOutput(ofproto.OFPP_CONTROLLER,
ofproto.OFPCML_NO_BUFFER)]
self.add_flow(datapath, 0, match, actions)
def add_flow(self, datapath, priority, match, actions, buffer_id=None):
ofproto = datapath.ofproto
parser = datapath.ofproto_parser
inst = [parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS,
actions)]
if buffer_id:
mod = parser.OFPFlowMod(datapath=datapath, buffer_id=buffer_id,
priority=priority, match=match,
instructions=inst)
else:
mod = parser.OFPFlowMod(datapath=datapath, priority=priority,
match=match, instructions=inst)
datapath.send_msg(mod)
#set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
def _packet_in_handler(self, ev):
# If you hit this you might want to increase
# the "miss_send_length" of your switch
if ev.msg.msg_len < ev.msg.total_len:
self.logger.debug("packet truncated: only %s of %s bytes",
ev.msg.msg_len, ev.msg.total_len)
msg = ev.msg
datapath = msg.datapath
ofproto = datapath.ofproto
parser = datapath.ofproto_parser
in_port = msg.match['in_port']
pkt = packet.Packet(msg.data)
eth = pkt.get_protocols(ethernet.ethernet)[0]
if eth.ethertype == ether_types.ETH_TYPE_LLDP:
# ignore lldp packet
return
dst = eth.dst
src = eth.src
dpid = datapath.id
self.mac_to_port.setdefault(dpid, {})
self.logger.info("packet in %s %s %s %s", dpid, src, dst, in_port)
# learn a mac address to avoid FLOOD next time.
self.mac_to_port[dpid][src] = in_port
if dst in self.mac_to_port[dpid]:
out_port = self.mac_to_port[dpid][dst]
else:
out_port = ofproto.OFPP_FLOOD
actions = [parser.OFPActionOutput(out_port)]
# install a flow to avoid packet_in next time
if out_port != ofproto.OFPP_FLOOD:
match = parser.OFPMatch(in_port=in_port, eth_dst=dst)
# verify if we have a valid buffer_id, if yes avoid to send both
# flow_mod & packet_out
if msg.buffer_id != ofproto.OFP_NO_BUFFER:
self.add_flow(datapath, 1, match, actions, msg.buffer_id)
return
else:
self.add_flow(datapath, 1, match, actions)
data = None
if msg.buffer_id == ofproto.OFP_NO_BUFFER:
data = msg.data
out = parser.OFPPacketOut(datapath=datapath, buffer_id=msg.buffer_id,
in_port=in_port, actions=actions, data=data)
datapath.send_msg(out)
This simple_switch_13.py app only handles layer 2 forwarding, which is your case.
As you can see, after the connection established, the switch_features_handler will listen on this event and add a send all flow to controller flow on the switch.(table-miss flow)
And for the normal states, when the controller receives PACKET_IN, it will check if the dst_MAC is in the mac_to_port. If yes, then output to the port, and at the same time insert a flow(whose match field is inport and dst_MAC); else(not in the array), the action is set to be FLOOD by assigning the outport=FLOOD.
That's the case in Layer 2 switching.
For ICMP messages handling in layer 3 switching, you need to read the rest_router.py code, which is a lot more complicated.
You get ICMP Host Destination Unreachable because the ARP request is never answered by h2.
Since h1 gets no ARP reply, ICMP error message comes from its own IP stack.