I am trying to write an array into Excel. The code snippet is below.
import win32com.client as win32
import sys
import numpy as np
#--------------
if name=="main":
name="nlve9.xlsm"
excel=win32.Dispatch('Excel.Application')
sheet=excel.Workbooks(name).Worksheets('NLVE')
#--------------
testress=np.zeros(5000)
for i in range(0,5000):
testress[i]=float(i)
sheet.Range("AC18:AC5016").value=testress
#excel.ScreenUpdating = False
#for i in range(18,5017):
# sheet.Range("AC" + str(i)).value=testress[i-18]
#excel.ScreenUpdating = True
sys.exit()
When this runs, I get a column of zero the length of testress. When I replace the last line with below it works but it is excruciatingly slow. THs is part of an optimization problem so this will run hundreds of times. Hence, I need this to be fast.
for i in range(18,5017):
# sheet.Range("AC" + str(i)).value=testress[i-18]
What am I doing wrong with the first method(sheet.Range("AC18:AC5016").value=testress)?
If you are using the Range.Value property to set an array, Excel needs a 2D array of row & column values, even if your data is a 1D array.
[[r1c1,r1c2,...],[r2c1,r2c2,...] ...]
As an example:
import win32com.client as wc
xl = wc.gencache.EnsureDispatch('Excel.Application')
xl.Visible = True
wb = xl.Workbooks.Add()
sh = wb.Sheets[1]
sh.Range('A1:A10').Value = [[i] for i in range(10)]
yielding:
EDIT:
From the OP's code, change:
testress=np.zeros(5000)
for i in range(0,5000):
testress[i]=float(i)
sheet.Range("AC18:AC5016").value=testress
to:
rowCount = 5000
testress = [[i] for i in range(rowCount)]
sheet.Range('AC18:AC'+str(18+rowCount-1)).Value = testress
I'm trying to code a simple program for a ESP32 board.
My main program is fairly simple and it has to run on a loop.
On the side, the device also needs to be able to respond to HTTP requests with a very simple response.
This is my attempt (a rework of https://randomnerdtutorials.com/micropython-esp32-esp8266-bme280-web-server/):
try:
import usocket as socket
except:
import socket
from micropython import const
import time
REFRESH_DELAY = const(60000) #millisecondi
def do_connect():
import network
wlan = network.WLAN(network.STA_IF)
wlan.active(True)
if not wlan.isconnected():
print('connecting to network...')
wlan.config(dhcp_hostname=HOST)
wlan.connect('SSID', 'PSWD')
while not wlan.isconnected():
pass
print('network config:', wlan.ifconfig())
import json
import esp
esp.osdebug(None)
import gc
gc.collect()
do_connect()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, SENSOR_SCKT_PORT))
s.listen(5)
prevRun = 0
i = 0
while True:
print("iteration #"+str(i))
i += 1
# run every 60 seconds
curRun = int(round(time.time() * 1000))
if curRun - prevRun >= REFRESH_DELAY:
prevRun = curRun
# MAIN PROGRAM
# ......
# whole bunch of code
# ....
# run continuously:
try:
if gc.mem_free() < 102000:
gc.collect()
conn, addr = s.accept()
conn.settimeout(3.0)
print('Got a connection from %s' % str(addr))
request = conn.recv(1024)
conn.settimeout(None)
request = str(request)
#print('Content = %s' % request)
measurements = 'some json stuff'
conn.send('HTTP/1.1 200 OK\n')
conn.send('Content-Type: text/html\n')
conn.send('Connection: close\n\n')
conn.send(measurements)
conn.close()
except OSError as e:
conn.close()
print('Connection closed')
what happens is I only get the iteration #0, and then the while True loop halts.
If I ping this server with a HTTP request, I get a correct response, AND the loop advances to iteration #1 and #2 (no idea why it thinks I pinged it with 2 requests).
So it seems that socket.listen(5) is halting the while loop.
Is there any way to avoid this?
Any other solution?
I don't think that threading is an option here.
The problem is that s.accept() is a blocking call...it won't return until it receives a connection. This is why it pauses your loop.
The easiest solution is probably to check whether or not a connection is waiting before calling s.accept(); you can do this using either select.select or select.poll. I prefer the select.poll API, which would end up looking something like this:
import esp
import gc
import json
import machine
import network
import select
import socket
import time
from micropython import const
HOST = '0.0.0.0'
SENSOR_SCKT_PORT = const(1234)
REFRESH_DELAY = const(60000) # milliseconds
def wait_for_connection():
print('waiting for connection...')
wlan = network.WLAN(network.STA_IF)
while not wlan.isconnected():
machine.idle()
print('...connected. network config:', wlan.ifconfig())
esp.osdebug(None)
gc.collect()
wait_for_connection()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, SENSOR_SCKT_PORT))
s.listen(5)
poll = select.poll()
poll.register(s, select.POLLIN)
prevRun = 0
i = 0
while True:
print("iteration #"+str(i))
i += 1
# run every 60 seconds
curRun = int(round(time.time() * 1000))
if curRun - prevRun >= REFRESH_DELAY:
prevRun = curRun
# MAIN PROGRAM
# ......
# whole bunch of code
# ....
# run continuously:
try:
if gc.mem_free() < 102000:
gc.collect()
events = poll.poll(100)
if events:
conn, addr = s.accept()
conn.settimeout(3.0)
print('Got a connection from %s' % str(addr))
request = conn.recv(1024)
conn.settimeout(None)
request = str(request)
# print('Content = %s' % request)
measurements = 'some json stuff'
conn.send('HTTP/1.1 200 OK\n')
conn.send('Content-Type: text/html\n')
conn.send('Connection: close\n\n')
conn.send(measurements)
conn.close()
except OSError:
conn.close()
print('Connection closed')
You'll note that I've taken a few liberties with your code to get it running on my device and to appease my sense of style; primarily, I've excised most of your do_connect method and put all the imports at the top of the file.
The only real changes are:
We create a select.poll() object:
poll = select.poll()
We ask it to monitor the s variable for POLLIN events:
poll.register(s, select.POLLIN)
We check if any connections are pending before attempting to handle a connection:
events = poll.poll(100)
if events:
conn, addr = s.accept()
conn.settimeout(3.0)
[...]
With these changes in place, running your code and making a request looks something like this:
iteration #0
iteration #1
iteration #2
iteration #3
iteration #4
iteration #5
iteration #6
Got a connection from ('192.168.1.169', 54392)
iteration #7
iteration #8
iteration #9
iteration #10
Note that as written here, your loop will iterate at least once every 100ms (and you can control that by changing the timeout on our call to poll.poll()).
Note: the above was tested on an esp8266 device (A Wemos D1 clone) running MicroPython v1.13-268-gf7aafc062).
I have been tasked with making plots of winds at various levels of the atmosphere to support aviation. While I have been able to make some nice plots using GFS model data (see code below), I'm really having to make a rough approximation of height using pressure coordinates available from the GFS. I'm using winds at 300 hPA, 700 hPA, and 925 hPA to make an approximation of the winds at 30,000 ft, 9000 ft, and 3000 ft. My question is really for those out there who are metpy gurus...is there a way that I can interpolate these winds to a height surface? It sure would be nice to get the actual winds at these height levels! Thanks for any light anyone can share on this subject!
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
import numpy as np
from netCDF4 import num2date
from datetime import datetime, timedelta
from siphon.catalog import TDSCatalog
from siphon.ncss import NCSS
from PIL import Image
from matplotlib import cm
# For the vertical levels we want to grab with our queries
# Levels need to be in Pa not hPa
Levels = [30000,70000,92500]
# Time deltas for days
Deltas = [1,2,3]
#Deltas = [1]
# Levels in hPa for the file names
LevelDict = {30000:'300', 70000:'700', 92500:'925'}
# The path to where our banners are stored
impath = 'C:\\Users\\shell\\Documents\\Python Scripts\\Banners\\'
# Final images saved here
imoutpath = 'C:\\Users\\shell\\Documents\\Python Scripts\\TVImages\\'
# Quick function for finding out which variable is the time variable in the
# netCDF files
def find_time_var(var, time_basename='time'):
for coord_name in var.coordinates.split():
if coord_name.startswith(time_basename):
return coord_name
raise ValueError('No time variable found for ' + var.name)
# Function to grab data at different levels from Siphon
def grabData(level):
query.var = set()
query.variables('u-component_of_wind_isobaric', 'v-component_of_wind_isobaric')
query.vertical_level(level)
data = ncss.get_data(query)
u_wind_var = data.variables['u-component_of_wind_isobaric']
v_wind_var = data.variables['v-component_of_wind_isobaric']
time_var = data.variables[find_time_var(u_wind_var)]
lat_var = data.variables['lat']
lon_var = data.variables['lon']
return u_wind_var, v_wind_var, time_var, lat_var, lon_var
# Construct a TDSCatalog instance pointing to the gfs dataset
best_gfs = TDSCatalog('http://thredds-jetstream.unidata.ucar.edu/thredds/catalog/grib/'
'NCEP/GFS/Global_0p5deg/catalog.xml')
# Pull out the dataset you want to use and look at the access URLs
best_ds = list(best_gfs.datasets.values())[1]
#print(best_ds.access_urls)
# Create NCSS object to access the NetcdfSubset
ncss = NCSS(best_ds.access_urls['NetcdfSubset'])
print(best_ds.access_urls['NetcdfSubset'])
# Looping through the forecast times
for delta in Deltas:
# Create lat/lon box and the time(s) for location you want to get data for
now = datetime.utcnow()
fcst = now + timedelta(days = delta)
timestamp = datetime.strftime(fcst, '%A')
query = ncss.query()
query.lonlat_box(north=78, south=45, east=-90, west=-220).time(fcst)
query.accept('netcdf4')
# Now looping through the levels to create our plots
for level in Levels:
u_wind_var, v_wind_var, time_var, lat_var, lon_var = grabData(level)
# Get actual data values and remove any size 1 dimensions
lat = lat_var[:].squeeze()
lon = lon_var[:].squeeze()
u_wind = u_wind_var[:].squeeze()
v_wind = v_wind_var[:].squeeze()
#converting to knots
u_windkt= u_wind*1.94384
v_windkt= v_wind*1.94384
wspd = np.sqrt(np.power(u_windkt,2)+np.power(v_windkt,2))
# Convert number of hours since the reference time into an actual date
time = num2date(time_var[:].squeeze(), time_var.units)
print (time)
# Combine 1D latitude and longitudes into a 2D grid of locations
lon_2d, lat_2d = np.meshgrid(lon, lat)
# Create new figure
#fig = plt.figure(figsize = (18,12))
fig = plt.figure()
fig.set_size_inches(26.67,15)
datacrs = ccrs.PlateCarree()
plotcrs = ccrs.LambertConformal(central_longitude=-150,
central_latitude=55,
standard_parallels=(30, 60))
# Add the map and set the extent
ax = plt.axes(projection=plotcrs)
ext = ax.set_extent([-195., -115., 50., 72.],datacrs)
ext2 = ax.set_aspect('auto')
ax.background_patch.set_fill(False)
# Add state boundaries to plot
ax.add_feature(cfeature.STATES, edgecolor='black', linewidth=2)
# Add geopolitical boundaries for map reference
ax.add_feature(cfeature.COASTLINE.with_scale('50m'))
ax.add_feature(cfeature.OCEAN.with_scale('50m'))
ax.add_feature(cfeature.LAND.with_scale('50m'),facecolor = '#cc9666', linewidth = 4)
if level == 30000:
spdrng_sped = np.arange(30, 190, 2)
windlvl = 'Jet_Stream'
elif level == 70000:
spdrng_sped = np.arange(20, 100, 1)
windlvl = '9000_Winds_Aloft'
elif level == 92500:
spdrng_sped = np.arange(20, 80, 1)
windlvl = '3000_Winds_Aloft'
else:
pass
top = cm.get_cmap('Greens')
middle = cm.get_cmap('YlOrRd')
bottom = cm.get_cmap('BuPu_r')
newcolors = np.vstack((top(np.linspace(0, 1, 128)),
middle(np.linspace(0, 1, 128))))
newcolors2 = np.vstack((newcolors,bottom(np.linspace(0,1,128))))
cmap = ListedColormap(newcolors2)
cf = ax.contourf(lon_2d, lat_2d, wspd, spdrng_sped, cmap=cmap,
transform=datacrs, extend = 'max', alpha=0.75)
cbar = plt.colorbar(cf, orientation='horizontal', pad=0, aspect=50,
drawedges = 'true')
cbar.ax.tick_params(labelsize=16)
wslice = slice(1, None, 4)
ax.quiver(lon_2d[wslice, wslice], lat_2d[wslice, wslice],
u_windkt[wslice, wslice], v_windkt[wslice, wslice], width=0.0015,
headlength=4, headwidth=3, angles='xy', color='black', transform = datacrs)
plt.savefig(imoutpath+'TV_UpperAir'+LevelDict[level]+'_'+timestamp+'.png',bbox_inches= 'tight')
# Now we use Pillow to overlay the banner with the appropriate day
background = Image.open(imoutpath+'TV_UpperAir'+LevelDict[level]+'_'+timestamp+'.png')
im = Image.open(impath+'Banner_'+windlvl+'_'+timestamp+'.png')
# resize the image
size = background.size
im = im.resize(size,Image.ANTIALIAS)
background.paste(im, (17, 8), im)
background.save(imoutpath+'TV_UpperAir'+LevelDict[level]+'_'+timestamp+'.png','PNG')
Thanks for the question! My approach here is for each separate column to interpolate the pressure coordinate of GFS-output Geopotential Height onto your provided altitudes to estimate the pressure of each height level for each column. Then I can use that pressure to interpolate the GFS-output u, v onto. The GFS-output GPH and winds have very slightly different pressure coordinates, which is why I interpolated twice. I performed the interpolation using MetPy's interpolate.log_interpolate_1d which performs a linear interpolation on the log of the inputs. Here is the code I used!
from datetime import datetime
import numpy as np
import metpy.calc as mpcalc
from metpy.units import units
from metpy.interpolate import log_interpolate_1d
from siphon.catalog import TDSCatalog
gfs_url = 'https://tds.scigw.unidata.ucar.edu/thredds/catalog/grib/NCEP/GFS/Global_0p5deg/catalog.xml'
cat = TDSCatalog(gfs_url)
now = datetime.utcnow()
# A shortcut to NCSS
ncss = cat.datasets['Best GFS Half Degree Forecast Time Series'].subset()
query = ncss.query()
query.var = set()
query.variables('u-component_of_wind_isobaric', 'v-component_of_wind_isobaric', 'Geopotential_height_isobaric')
query.lonlat_box(north=78, south=45, east=-90, west=-220)
query.time(now)
query.accept('netcdf4')
data = ncss.get_data(query)
# Reading in the u(isobaric), v(isobaric), isobaric vars and the GPH(isobaric6) and isobaric6 vars
# These are two slightly different vertical pressure coordinates.
# We will also assign units here, and this can allow us to go ahead and convert to knots
lat = units.Quantity(data.variables['lat'][:].squeeze(), units('degrees'))
lon = units.Quantity(data.variables['lon'][:].squeeze(), units('degrees'))
iso_wind = units.Quantity(data.variables['isobaric'][:].squeeze(), units('Pa'))
iso_gph = units.Quantity(data.variables['isobaric6'][:].squeeze(), units('Pa'))
u = units.Quantity(data.variables['u-component_of_wind_isobaric'][:].squeeze(), units('m/s')).to(units('knots'))
v = units.Quantity(data.variables['v-component_of_wind_isobaric'][:].squeeze(), units('m/s')).to(units('knots'))
gph = units.Quantity(data.variables['Geopotential_height_isobaric'][:].squeeze(), units('gpm'))
# Here we will select our altitudes to interpolate onto and convert them to geopotential meters
altitudes = ([30000., 9000., 3000.] * units('ft')).to(units('gpm'))
# Now we will interpolate the pressure coordinate for model output geopotential height to
# estimate the pressure level for our given altitudes at each grid point
pressures_of_alts = np.zeros((len(altitudes), len(lat), len(lon)))
for ilat in range(len(lat)):
for ilon in range(len(lon)):
pressures_of_alts[:, ilat, ilon] = log_interpolate_1d(altitudes,
gph[:, ilat, ilon],
iso_gph)
pressures_of_alts = pressures_of_alts * units('Pa')
# Similarly, we will use our interpolated pressures to interpolate
# our u and v winds across their given pressure coordinates.
# This will provide u, v at each of our interpolated pressure
# levels corresponding to our provided initial altitudes
u_at_levs = np.zeros((len(altitudes), len(lat), len(lon)))
v_at_levs = np.zeros((len(altitudes), len(lat), len(lon)))
for ilat in range(len(lat)):
for ilon in range(len(lon)):
u_at_levs[:, ilat, ilon], v_at_levs[:, ilat, ilon] = log_interpolate_1d(pressures_of_alts[:, ilat, ilon],
iso_wind,
u[:, ilat, ilon],
v[:, ilat, ilon])
u_at_levs = u_at_levs * units('knots')
v_at_levs = v_at_levs * units('knots')
# We can use mpcalc to calculate a wind speed array from these
wspd = mpcalc.wind_speed(u_at_levs, v_at_levs)
I was able to take my output from this and coerce it into your plotting code (with some unit stripping.)
Your 300-hPa GFS winds
My "30000-ft" GFS winds
Here is what my interpolated pressure fields at each estimated height level look like.
Hope this helps!
I am not sure if this is what you are looking for (I am very new to Metpy), but I have been using the metpy height_to_pressure_std(altitude) function. It puts it in units of hPa which then I convert to Pascals and then a unitless value to use in the Siphon vertical_level(float) function.
I don't think you can use metpy functions to convert height to pressure or vice versus in the upper atmosphere. There errors are too when using the Standard Atmosphere to convert say pressure to feet.
I am trying to write an indicator originally from MT4 into NT7.
I have the following calculations in MT4:
dayi = iBarShift(Symbol(), myPeriod, Time[i], false);
Q = (iHigh(Symbol(), myPeriod,dayi+1) - iLow(Symbol(),myPeriod,dayi+1));
L = iLow(NULL,myPeriod,dayi+1);
H = iHigh(NULL,myPeriod,dayi+1);
O = iOpen(NULL,myPeriod,dayi+1);
C = iClose(NULL,myPeriod,dayi+1);
myperiod is a variable where I place the period in minutes (1440 = 1day).
What are the equivalent functions in NT7 to iBarShift, iHigh and so on?
Thanks in advance
For NinjaTrader:
iLow = Low or Lows for multi-time frame
iHigh = High or Highs
iOpen = Open or Opens
iClose = Close or Closes
So an example would be
double low = Low[0]; // Gets the low of the bar at index 0, or the last fully formed bar (If CalculateOnBarClose = true)
In order to make sure you are working on the 1440 minute time frame, you will need to add the following in the Initialize() method:
Add(PeriodType.Minute, 1440);
If there are no Add statements prior to this one, it will place it at index 1 (O being the chart default index) in a 2 dimensional array. So to access the low of the 1440 minute bar at index 0 would be:
double low = Lows[1][0];
For iBarShift look at
int barIndex = Bars.GetBar(time);
which will give you the index of the bar with the matching time. If you need to use this function on the 1440 bars (or other ones), use the BarsArray property to access the correct Bar object and then use the GetBar method on it. For example:
int barIndex = BarsArray[1].GetBar(time);
Hope that helps.
In a ModelForm I can write a clean_<field_name> member function to automatically validate and clean up data entered by a user, but what can I do about dirty json or csv files (fixtures) during a manage.py loaddata?
Fixtures loaded with loaddata are assumed to contain clean data that doen't need validation (usually as an inverse operation to a prior dumpdata), so the short answer is that loaddata isn't the approach you want if you need to clean your inputs.
However, you probably can use some of the underpinnings of loaddata while implementing your custom data cleaning code--I'm sure you can easily script something using the Django serialization libs to read your existing data files them in and the save the resulting objects normally after the data has been cleaned up.
In case others want to do something similar, I defined a model method to do the cleaning (so it can be called from ModelForms)
MAX_ZIPCODE_DIGITS = 9
MIN_ZIPCODE_DIGITS = 5
def clean_zip_code(self, s=None):
#s = str(s or self.zip_code)
if not s: return None
s = re.sub("\D","",s)
if len(s)>self.MAX_ZIPCODE_DIGITS:
s = s[:self.MAX_ZIPCODE_DIGITS]
if len(s) in (self.MIN_ZIPCODE_DIGITS-1,self.MAX_ZIPCODE_DIGITS-1):
s = '0'+s # FIXME: deal with other intermediate lengths
if len(s)>=self.MAX_ZIPCODE_DIGITS:
s = s[:self.MIN_ZIPCODE_DIGITS]+'-'+s[self.MIN_ZIPCODE_DIGITS:]
return s
Then wrote a standalone python script to clean up my legacy json files using any clean_ methods found among the models.
import os, json
def clean_json(app = 'XYZapp', model='Entity', fields='zip_code', cleaner_prefix='clean_'):
# Set the DJANGO_SETTINGS_MODULE environment variable.
os.environ['DJANGO_SETTINGS_MODULE'] = app+".settings"
settings = __import__(app+'.settings').settings
models = __import__(app+'.models').models
fpath = os.path.join( settings.SITE_PROJECT_PATH, 'fixtures', model+'.json')
if isinstance(fields,(str,unicode)):
fields = [fields]
Ns = []
for field in fields:
try:
instance = getattr(models,model)()
except AttributeError:
print 'No model named %s could be found'%(model,)
continue
try:
cleaner = getattr(instance, cleaner_prefix+field)
except AttributeError:
print 'No cleaner method named %s.%s could be found'%(model,cleaner_prefix+field)
continue
print 'Cleaning %s using %s.%s...'%(fpath,model,cleaner.__name__)
fin = open(fpath,'r')
if fin:
l = json.load(fin)
before = len(l)
cleans = 0
for i in range(len(l)):
if 'fields' in l[i] and field in l[i]['fields']:
l[i]['fields'][field]=cleaner(l[i]['fields'][field]) # cleaner returns None to delete records
cleans += 1
fin.close()
after = len(l)
assert after>.5*before
Ns += [(before, after,cleans)]
print 'Writing %d/%d (new/old) records after %d cleanups...'%Ns[-1]
with open(fpath,'w') as fout:
fout.write(json.dumps(l,indent=2,sort_keys=True))
return Ns
if __name__ == '__main__':
clean_json()