I have managed to get a live raw bytearray file containing just the data. I tried converting it into wave file using the following code:
import scipy.io.wavfile
import numpy as np
import soundfile as sf
def convert_bytes_to_audio():
fi = open("receive_data.txt","r")
for line in fi:
scipy.io.wavfile.write('format.wav', 48000, np.frombuffer(str.encode(line),dtype = 'B'))
fi.close()
def main():
convert_bytes_to_audio()
if __name__ == '__main__':
convert_bytes_to_audio()
The bytearray:
b'\x00Z\x15\x02\xf9\x15q\x9d1$\xd2\xde:\xb44\xaf\x8d_4\xbe\xe5\x02\x1f\x87\xe0,d\xa8YS\x02\xea\xdc\xf8\x85\xcc\x87p\x8be)Fi4FE\xeb\xf5\x10\xfbr\xd3-)\x02J\xd9\xebOo\xca\xd3\x04\xcaaG\xf2\x17\xe6\x04\xd3\xdds(R\x98\xf8\xe6\x80#b\xa3\x0e~h\xb1\x02\xf7"(%\x12y\xb6\x1e/wL\n\x9a\xb39\xd7\xb2&Mc\xca\xdc\x9f\xbb\xff\xa7\x81n\xe213\x85\xea\xc8p\x1c{\x19j\x1a\xdd\x0f\xdd\x9d\xde\x19s\x89]\xf8z\x11R\xed5\xd3g\xacy\x85\xa8\x8f\xca\xad\x91\xba\xbe\x07\xa5\xfa\xa0'
b"\x01Z\x8c\xe3\xd9\xd82N8F\x1a\x9d\r\xdct\x97+\xdf\x7f8\x1dL\xb6\xae\xdb\xc2\xbc\xd4\x8b\x1f\xe7rZ\rI\x85\x7f\xb6F\x85\x15\xc6$\xea\x82&l\r8\x82Zu\x00#\x0b\x81\xe6ij\xeb\x02\xbf[\xb6\xfa\x83\x111\x86\xd0\xf2\xb6\x0f\x94\xedk\x81\x0c\x1b%\x16\xd9\xb31\x0c\x04:\x8c[O\xde'\xae[\xe0\x90W/\x9c\xea\xb4\xcf(}\x9e\x19\xa3\xdc\x16?\xc2\x01L[\x1d\x9c\xcc\x1bqP\x95g\xd2s\xbdNM\xe8p\x9c:rW\x01\xf8\xc9k\xf8V;\x9f\xe3\xb7\xe4\x9c\xd5f27\xe8Kg\xbb\xdf\xb0\xbc2\xf9\x0e<G$*\x91\xbd\xdb\x06\xe9\x9fd\x00\xe5\x8c\x88\xf8R\x93\x91\xab\x84\xc7\xa2\x93\x91\x8c#\x9e\xfd\x1a\xf9FM\xd2\xe9\\\x80\xa5\x18\xa5\xec\xb1\xf8u[\x1a\xe0\xd6\xfda\x07r)\xa9\xf4l\xab\xcf\x15'\xd5\xd41\n\x0c]\x045\x03\r\xde\xc5\xbb;74)\xbe;\x06\x06\xe4{\xbf\x8b\xf1O\xfb\xea7\xbf\x16#(!w\xa8`,\x7f\xc0"
b'\x02Z\x88\xe0f+.\x18 8MG\xa2~_Z\xf4-\x11\x03\xf2w\xf5\x8d\x1d}sCD\xf0\xe9\x97\xf8\x8bg\xe1.0\xf9\x05~\xa4\x17\xf6\x92"\xa7l|\xc8,\xe8>\x14\xca\xb4\x01\x14\x82\xe8,\xcb|\nS\xe8$\xf9\xb5A \xf7\xa9\x9co\n\x19\xda\xcf\xa0\x8c\x97\x9av\xf4\xeb\xd0uw\xf5\xbcQj\xf5\x1b\xca\x0eA\x85\x82\xa60\x81\xcb\x03k\xcf\xa1.#\xee\xa53R\xd0\x87F\x16\xa8\xf3\xa7I\xc4|sQ\x15T\x08R\xdd\x9c\x02\\Y\xb9u\xcb\x80\xea\xbc\xfd\x18O6\xce#\x00\xd5\x15\xa5\xab\r\xf0\x8e6)O\xb8\xd8\xac\xf4\x87x\xdc\x08)\xfc\x9c\xff\xbd\xd0|\xb5kg#\xaa\xa5Gc\xfd\x1c\x1c\xa1\xc7Zl\xb5\xc8\xb3\x07\xa4\xf1]\xa2\xc5\xfb\xd9{]\xffG0\xcc&\xcf\xc8\xbfB*gG\xff\x1fI\x14\x1bs\xa6][\xcea\x11\x7f\x8d\x11\xf9\xdc\xa1\xfcQ\xce\xca\x14\x8d\\\xfd\x08\x9e\x1bD\xa7\x84\x9b5\xb5/\xac->~\x94\x1c\xd2\x83\x08_\\[\x07P\xbf\xc4\xea '
b'\x03Z\x8d\xea\xc5(\xcf\x1e\xa1\x07\x14\xf2\xd7\xe8\x1e\x82d\xe9I\x8fN,(\xd5]\xc4\x83\x14\xe9+\xfb\x93\xfb\xc1\x9e\xf4\xa9l\x95\xbf\x88\x88\xc7\xac\xdd[\xce6\x0bG+gq\x17n[\xb6\x17M\x12\r\x16!"{7M\xd3\xda,4\x9fE\x90\xcd\xae*\xdcf\x14?\xd2=\x00\xf5\x0f+*\xb8Sc|\x00\x86\xed7\xc3\x8e)\xe7\x0c\xe4\xfcml0\x9d\xbd\x03\x90)\xb1\xb2t>S=\xc7\xd3\xe16\xbb\x8f\xbf\xaf\x1c\x8bp>\xa1\xf9\x91\x90GH\xd5|^\x1f\x1d?\x95\xd2\x84\xe4\xc8\xc6\xf3\xc8Si\x05\x95\xbec\xd8{3O\x11M\xb1\x1fL<\x1cR\xdaI\xdc3Hh(\x0e\xd9\xf6\x82\x05\xd3\x9c\x9d\x16\x1e\xf6\'`\xee\xc6\xf2\x1e\xcb\xa1]\xcd\x0b\xff\xaaT\xd7\xbf\xed\x922$\xac\x1aY\xd8ue`\x9c\x0e\x9a\x8b0U\xe42\x92\x00\xe8d\tt\xd7X\x1a]\x9e\xc1\xc5+N\xea\xe3\xc6s\xfd\x86\x8c\x12\x80[\xc9\xaap\x99\x99w\xccX>\xcf[\x1e\xad\x1f0\x03\xe2\xc0'
b"\x04Zo\xc4\x9c\xceL]=S\xd7\xce\xd9cd\t\xe1\n\x01J\xd5\xee\xe93R\xe6\xf2\xbd\x89u P\xafX\xdaDH\x13$\xff^\x1bK\xa6\xb3\x90\xeeq\x94}\x8a-\xf3c\x7f\x7f\xeaF\x06#\x9a\xf7\xd2a\x17\x99\xf5\xf1\x92\xa3\x88\x06\x9a\xe5qn\x82\xd4R\xa3C\xc6\x08Y\x07b_\xf2\x1a\xf1\xe3}r\xde6\xebH\xf4\x1c\x08\xfc\xeaT/A\xb9\x98f\xee\x0b\x95%c\x9e\xe0i\xe0Q\xc5\xff4g\x01\xcf\xc4\xe0c\xc3r\xb1\xf7\xab\x8e5I\t/C\xb3=a\xca\xd0\xb1\xebh#\x88m\xcdt(\x99\xa0\xc6\xb9\xb3\xb5\xee\xf8\x1f\xd3\xe2\x9c\xad'D\xb1\xd5_[\xb0\xe9\xb2\x8dm\xa3*\xae5\xbfB\xf7\xf9\xb5\xd8\x01u\xbdg\xdd\xa2\xc1\xf0\xc2\x9a\x89P\xb0\x87\x0e\xd6\x9a\xf5\xcb\xfc\xe8\x05QKe4\x0f\x8c\x8eH\x9e\xf7\xf6\xbeF\xc7(\x9d\x8a\xe4O#\x91\xdd\xc2\xa9\x8e\xbfv\xf0D\xb4\x8b\xa0"
b'\x05Zv\xe98m|Y\xd0^\x19\xdf#s\xb0\xd3\x11\xb7a]4\xaf\xff\x16K\xea\xea\xbf\xc7b8\xc16\xae\xe3\x81+\x0c]\x81\xb5c4Z\xed\xd9]\xdaPM\x15/\'5rZ\xb5D\xd9b\xe0dOuy\xdc\xb7\xd6^U.<\xa2\xf3\xfbdqH\x0f\xd3\x92O\x98\xd3\xcc\xd1\x1eb\xe4\x9f_V\x08\xfa\xf7\xa9#\xb6h[g\x8a\x08\xd2\x89&\x96b\xfd\xca\x06W\'7e\xbdG\x16\xac\x81X\xea<\xbc\xe8\xf8\xf3\xcd\xbf\x9a\x84M\x85w\xc9\x9fG`\xa4\xa5A?(\x0c,\xf2"\xeb\xc8\xe8\xcd\xc6\xcf\xdb\xc7HDh<\xd1\x0c`e\x90v\xe4\xe1$F\x1a\xf4\x0e\xc83~0\xc6q[\xe0#\xf82#\x0f\x94\xfc+8\x9a\x0f\xaa\xdb\xcf\xda\x85\x90T\xea\x9f\xff \xb6\x16\rZ\x92\xd0dY\x10\xa8\xed\xe7#F\xdc[k\x98\x9c\x8a\x11\xd6Z\x84\xbe\xef\xaf"\x08i\x9f\xf7\x94\x85\x07\xce\xa8.C\xaf\xac(\xc2\xae\x97quC\x95\xbb;\xdb\xbb\xa0'
b'\x06Z\x89\xef\xc9\x7f\x08\xc9\x0b\xac\xbd\x9a"?z\xb0\x002\xfb\xac\x8e\xeb\x89\x1b,\x89Ee>\x82\xad&\x8ft\xf8\x13y\x94\xcd\xd3\xe9\xa0\n\x16\xc0\xcc\x90\xc91%\xd4\xee\xd8\xcd_\xc9\x02\xd21\xf7\x13\xd4\xf7\xba]\x7f^\xcbuZ\x84\xdf%\xa3\x81\x1dw\xaf <^\x94Ng.\xf7\xae\x0f\xec\xbb\xc3\xb6\xbd\xf4\x0c\xea#7=\xba\x08>FZ\xa8%<\xce\xb8S\x8c\xa2\xee_\xc83}\xbb\xc5\x9d\x9f\x15\x14\xe0\xb8\xf9*0\xb8\x87\xca\xa1\x11Y\xde\x8b\xb5\x8e\x16\xb8\xc4rL\xc8&ut\x8e\x19\xf39)F\xcfR)\xe2\xa8\xff\x8b\xb4s\xc5}\xed%\x86\x13\xbc\x03\x12\xdcb\xa6\\E5\x98\x03\xe5%\x9f4\xb0\xac-\x8e\xac\x83\x17\x08\xb5\x0f\x18\xc3\x0e\xbf\xe5S\xa0\xeb9ni\xc9\xd6\r\xdd\x10\xcc&\'\xf5\xdbz\xcf\x1ac\xf0\xdb\xae\xe6D\x9f\x8cq\x14\xa0\xa7\xcfq!M\xc9\xe46\x19"\xbc=\x82\x18\x87\'\xe0>'
I managed to get the wav file but it is unclear and not exactly for the amount of time I recorded it.
Also the format is u8 type, and encoding is pcm_f32le. But I am unsure how to change those.
Generated wav file:
Can anyone help me on this.
Related
i'm trying to follow this tutorial with my own local data files:
CNTK tutorial
i have the following function to save my data array into a txt file feedable to CNTK:
# Save the data files into a format compatible with CNTK text reader
def savetxt(filename, ndarray):
dir = os.path.dirname(filename)
if not os.path.exists(dir):
os.makedirs(dir)
if not os.path.isfile(filename):
print("Saving", filename )
with open(filename, 'w') as f:
labels = list(map(' '.join, np.eye(11, dtype=np.uint).astype(str)))
for row in ndarray:
row_str = row.astype(str)
label_str = labels[row[-1]]
feature_str = ' '.join(row_str[:-1])
f.write('|labels {} |features {}\n'.format(label_str, feature_str))
else:
print("File already exists", filename)
i have 2 ndarrays of the following shape that i want to feed the model:
train.shape
(1976L, 15104L)
test.shape
(1976L, 15104L)
Then i try to implement the fucntion like this:
# Save the train and test files (prefer our default path for the data)
data_dir = os.path.join("C:/Users", 'myself', "OneDrive", "IA Project", 'data', 'train')
if not os.path.exists(data_dir):
data_dir = os.path.join("data", "IA Project")
print ('Writing train text file...')
savetxt(os.path.join(data_dir, "Train-128x118_cntk_text.txt"), train)
print ('Writing test text file...')
savetxt(os.path.join(data_dir, "Test-128x118_cntk_text.txt"), test)
print('Done')
and then i get the following error:
Writing train text file...
Saving C:/Users\A702628\OneDrive - Atos\Microsoft Capstone IA\Capstone data\train\Train-128x118_cntk_text.txt
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
<ipython-input-24-b53d3c69b8d2> in <module>()
6
7 print ('Writing train text file...')
----> 8 savetxt(os.path.join(data_dir, "Train-128x118_cntk_text.txt"), train)
9
10 print ('Writing test text file...')
<ipython-input-23-610c077db694> in savetxt(filename, ndarray)
12 for row in ndarray:
13 row_str = row.astype(str)
---> 14 label_str = labels[row[-1]]
15 feature_str = ' '.join(row_str[:-1])
16 f.write('|labels {} |features {}\n'.format(label_str, feature_str))
IndexError: list index out of range
Can somebody please tell me what's going wrong with this part of the code? And how could i fix it? Thank you very much in advance.
Since you're using your own input data -- are they labelled in the range 0 to 9? The labels array only has 10 entries in it, so that could cause an out-of-range problem.
I am trying to copy over S3 large dataset (larger than RAM) using SQL alchemy.
My constraints are:
I need to use sqlalchemy
I need to keep memory pressure at lowest
I don't want to use the local filsystem as intermediary step to send data to s3
I just want to pipe data from a DB to S3 in a memory efficient way.
I can to do it normal with data sets (using below logic) but with larger dataset I hit a buffer issue.
The first problem I solved is that executing a query usually buffers the result in memory. I use the fetchmany() method.
engine = sqlalchemy.create_engine(db_url)
engine.execution_options(stream_results=True)
results=engine.execute('SELECT * FROM tableX;')
while True:
chunk = result.fetchmany(10000)
if not chunk:
break
On the other side, I have a StringIO buffer that I feed with the fetchmany data check. Then I send its content to s3.
from io import StringIO
import boto3
import csv
s3_resource = boto3.resource('s3')
csv_buffer = StringIO()
csv_writer = csv.writer(csv_buffer, delimiter=';')
csv_writer.writerows(chunk)
s3_resource.Object(bucket, s3_key).put(Body=csv_buffer.getvalue())
The problem I have is essentially a design issue, how do I make these parts work together. Is it even possible in the same runtime?
engine = sqlalchemy.create_engine(db_url)
s3_resource = boto3.resource('s3')
csv_buffer = StringIO()
csv_writer = csv.writer(csv_buffer, delimiter=';')
engine.execution_options(stream_results=True)
results=engine.execute('SELECT * FROM tableX;')
while True:
chunk = result.fetchmany(10000)
csv_writer = csv.writer(csv_buffer, delimiter=';')
csv_writer.writerows(chunk)
s3_resource.Object(bucket, s3_key).put(Body=csv_buffer.getvalue())
if not chunk:
break
I can make it work for one cycle of fetchmany, but not several. Any idea?
I'm assuming that by "make these parts work together" you mean you want a single file in S3 instead of just parts? All you need to do is to create a file object that, when read, will issue a query for the next batch and buffer that. We can make use of python's generators:
def _generate_chunks(engine):
with engine.begin() as conn:
conn = conn.execution_options(stream_results=True)
results = conn.execute("")
while True:
chunk = results.fetchmany(10000)
if not chunk:
break
csv_buffer = StringIO()
csv_writer = csv.writer(csv_buffer, delimiter=';')
csv_writer.writerows(chunk)
yield csv_buffer.getvalue().encode("utf-8")
This is a stream of chunks of your file, so all we need to do is to stitch these together (lazily, of course) into a file object:
class CombinedFile(io.RawIOBase):
def __init__(self, strings):
self._buffer = ""
self._strings = iter(strings)
def read(self, size=-1):
if size < 0:
return self.readall()
if not self._buffer:
try:
self._buffer = next(self._strings)
except StopIteration:
pass
if len(self._buffer) > size:
ret, self._buffer = self._buffer[:size], self._buffer[size:]
else:
ret, self._buffer = self._buffer, b""
return ret
chunks = _generate_chunks(engine)
file = CombinedFile(chunks)
upload_file_object_to_s3(file)
Streaming the file object to S3 is left as an exercise for the reader. (You can probably use put_object.)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
d = {'a':'текст',
'b':{
'a':'текст2',
'b':'текст3'
}}
print(d)
w = open('log', 'w')
json.dump(d,w, ensure_ascii=False)
w.close()
It gives me:
UnicodeEncodeError: 'ascii' codec can't encode characters in position 1-5: ordinal not in range(128)
Post the full traceback, the error could be coming from the print statement when it fails to decode the dictionary object. For some reason print statement cannot decode all contents if you have Cyrillic text in it.
Here is how I save to json my dictionary that contains Cyrillics:
mydictionary = {'a':'текст'}
filename = "myoutfile"
with open(filename, 'w') as jsonfile:
json.dump(mydictionary, jsonfile, ensure_ascii=False)
The trick will be reading in json back into dictionary and doing things with it.
To read in json back into dictionary:
with open(filename, 'r') as jsonfile:
newdictonary = json.load(jsonfile)
Now when you look at the dictionary, the word 'текст' looks (encoded) like '\u0442\u0435\u043a\u0441\u0442'. You simply need to decode it using encode('utf-8'):
for key, value in newdictionary.iteritems():
print value.encode('utf-8')
Same goes for lists if your Cyrillic text is stored there:
for f in value:
print f.encode('utf-8')
# or if you plan to use the val somewhere else:
f = f.encode('utf-8')
i have this function: write_reversed_file(input_filename, output_filename) that writes to the given output file the contents of the given input file with the lines in reversed order. i just need the output to be written to the file (output_filename) rather than to the terminal (python shell).
the only part im missing is to store the output into the file.
i successfully managed to complete the reversing lines part.
def write_reversed_file(input_filename, output_filename):
for line in reversed(list(open(filename))):
print(line.rstrip())
def write_reversed_file(input_filename, output_filename):
s = ""
f = open(input_filename,"r")
lines = f.read().split("\n")
f.close()
for line in reversed(lines):
s+=line.rstrip()+"\n"
f = open(outPutFile.txt,"w")
f.write(s)
f.close()
It is good practice to use 'with open as' format when working with files since it is automatically closing the file for us. (as recommended in docs.python.org)
def write_reversed_file(input_filename, output_filename):
with open(output_filename, 'w') as f:
with open(input_filename, 'r') as r:
for line in reversed(list(r.read())):
f.write(line)
write_reversed_file("inputfile.txt", "outputfile.txt")
Hiya i have made a program that stores the player name and strength..Here is the code:
data = {
"PLAYER":name2,
"STRENGTH":str(round(strength, 2)),
}
with open("data2.txt", "w", encoding="utf-8") as file:
file.write(repr(data))
file.close()
So this stores the data so what to i do if i wanna append/change the value after a certain action usch as a 'BATTLE'
Is it possible the get the variable of 'STRENGTH' and then change the number?
At the moment to read data from the external file 'DATA1.txt'i am using this code:
with open("data1.txt", "r", encoding="utf-8") as file:
data_string = file.readline()
data = eval(data_string)
# (data["STRENGTH"])
S1 = (float(data["STRENGTH"]))
file.close()
Now i can do something with the variable --> 'S1'
Here is the external text file 'data1.txt'
{'PLAYER': 'Oreo', 'STRENGTH': '11.75'}
... But i wanna change the strength value after a "battle" many thanks
Maybe you're not understanding Python dict semantics?
Seems to me you're doing a lot of unnecessary things like S1 = (float(data['STRENGTH'])) to try to manipulate and change values when you could be doing really simple stuff.
>>> data = {'PLAYER': 'Oreo', 'STRENGTH': '11.75'}
>>> data['STRENGTH'] = float(data['STRENGTH'])
>>> data
{'PLAYER': 'Oreo', 'STRENGTH': 11.75}
>>> data['STRENGTH'] += 1
>>> data
{'PLAYER': 'Oreo', 'STRENGTH': 12.75}
Maybe you should give Native Data Types -- Dive Into Python 3 a read to see if it clears things up.