Turf returns wrong (?) bearing and distance - reactjs

In my react app I have this piece of code:
import * as turfBearing from '#turf/bearing'
import * as turfDistance from '#turf/distance'
distance( p1, p2 ) {
return Math.round( turfDistance.default( p1, p2 ) * 1000 )
}
bearing( p1, p2 ) {
return ( Math.round( turfBearing.default( p1, p2 ) ) + 360 ) % 360
}
Given the data:
const p1 = [ 48.1039072, 11.6558318 ]
const p2 = [ 48.1035817, 11.6555873 ]
The results are:
bearing = 233, dist = 45
If I feed the same data to an online service like https://www.omnicalculator.com/other/azimuth it gives results:
which are considerably different from turf's.
Is this turf's problem or online calculator's?

Indeed, Turfjs has some problems in calc. I swapped it for a randomly picked library "sgeo": "^0.0.6", and the code:
distance( p1, p2 ) {
return Math.round( new sgeo.latlon( ...p1 ).distanceTo( new sgeo.latlon( ...p2 ) ) * 1000 )
}
bearing( p1, p2 ) {
return Math.round( new sgeo.latlon( ...p1 ).bearingTo( new sgeo.latlon( ...p2 ) ) )
}
produces relevant results:
bearing = 206 dist = 40

Turf expects data in (longitude, latitude) order per the GeoJSON standard, see https://github.com/Turfjs/turf#data-in-turf
In your case input data should be:
const p1 = [ 11.6558318, 48.1039072 ]
const p2 = [ 11.6555873, 48.1035817 ]

Related

Inserting data from csv too slow Django

My problem is that inserting data into the database is too slow. I have written a correct algorithm that loads the data properly however with this method it will take 700 hours to load the database. There are almost 30 million records in the csv file.
Here is my models.py
from django.db import models
class Region(models.Model):
region = models.CharField(max_length=20)
class Rank(models.Model):
rank = models.IntegerField()
class Chart(models.Model):
chart = models.CharField(max_length=8)
class Artist(models.Model):
artist = models.CharField(max_length=60)
class Title(models.Model):
title = models.CharField(max_length=60)
class ArtistTitle(models.Model):
artist = models.ForeignKey(Artist, on_delete=models.CASCADE)
title = models.ForeignKey(Title, on_delete=models.CASCADE)
class SpotifyData(models.Model):
title = models.ForeignKey(ArtistTitle, related_name='re_title', on_delete=models.CASCADE)
rank = models.ForeignKey(Rank, on_delete=models.CASCADE)
date = models.DateField()
artist = models.ForeignKey(ArtistTitle, related_name='re_artist', on_delete=models.CASCADE)
region = models.ForeignKey(Region, on_delete=models.CASCADE)
chart = models.ForeignKey(Chart, on_delete=models.CASCADE)
streams = models.IntegerField()
My upload script looks like this:
def load_to_db(self, df):
bad = 0
good = 0
start = datetime.datetime.now
for _, row in df.iterrows():
try:
region_obj, _ = Region.objects.get_or_create(
region=row["region"],
)
rank_obj, _ = Rank.objects.get_or_create(
rank=row["rank"],
)
chart_obj, _ = Chart.objects.get_or_create(
chart=row["chart"],
)
artist_obj, _ = Artist.objects.get_or_create(
artist=row["artist"],
)
title_obj, _ = Title.objects.get_or_create(
title=row["title"],
)
arttit_obj, _ = ArtistTitle.objects.update_or_create(
artist=artist_obj,
title=title_obj,
)
spotifydata_obj, _ = SpotifyData.objects.update_or_create(
title=arttit_obj,
rank=rank_obj,
date=row["date"],
artist=arttit_obj,
region=region_obj,
chart=chart_obj,
streams=row["streams"],
)
good += 1
now = datetime.datetime.now
print(f"goods: {good}, loading time: {start-now}", )
except Exception as e:
bad += 1
current_time = datetime.datetime.now()
with open("data_load_logging.txt", "w") as bad_row:
bad_row.write(
f"Error message: {e} \n"
+ f"time: {current_time}, \n"
+ f"title: {row['title']}, type: {row['title']} \n"
+ f"rank: {int(row['rank'])}, type: {int(row['rank'])} \n"
+ f"date: {row['date']}, type: {row['date']} \n"
+ f"artist: {row['artist']}, type: {row['artist']} \n"
+ f"region: {row['region']}, type: {row['region']} \n"
+ f"chart: {row['chart']}, type: {row['chart']} \n"
+ f"streams: {int(row['streams'])}, type: {int(row['streams'])} \n"
+ "-" * 30
+ "\n"
)
I know it could probably help to use bulk/bulk_create/bulk_update but I can't figure out how to write the correct script....
def load_to_db(self, path):
start_time = timezone.now()
try:
with open(path, "r") as csv_file:
data = csv.reader(csv_file)
next(data)
packet_region = []
packet_rank = []
packet_chart = []
packet_artist = []
packet_title = []
packet_artist_title = []
packet_spotify_data = []
bad = -1 # first row is a header
for row in data:
region = Region(
region = row[4]
)
rank = Rank(
rank = row[1]
)
chart = Chart(
chart = row[5]
)
artist = Artist(
artist = row[3]
)
title = Title(
title = row[0]
)
artist_title = ArtistTitle(
artist = artist,
title = title
)
spotify_data = SpotifyData(
title = artist_title,
rank = rank,
date = row[3],
artist = artist_title,
region = region,
chart = chart,
streams = int(row[6])
)
packet_region.append(region)
packet_rank.append(rank)
packet_chart.append(chart)
packet_artist.append(artist)
packet_title.append(title)
packet_artist_title.append(artist_title)
packet_spotify_data.append(spotify_data)
if len(packet_spotify_data) > 1000:
print(datetime.datetime.now())
Region.objects.bulk_create(packet_region)
Rank.objects.bulk_create(packet_rank)
Chart.objects.bulk_create(packet_chart)
Artist.objects.bulk_create(packet_artist)
Title.objects.bulk_create(packet_title)
ArtistTitle.objects.bulk_update(packet_artist_title)
SpotifyData.objects.bulk_update(packet_spotify_data)
packet_region = []
packet_rank = []
packet_chart = []
packet_artist = []
packet_title = []
packet_artist_title = []
packet_spotify_data = []
logging.info(f"Failure numbers: {bad}")
if packet_spotify_data:
Region.objects.bulk_create(packet_region)
Rank.objects.bulk_create(packet_rank)
Chart.objects.bulk_create(packet_chart)
Artist.objects.bulk_create(packet_artist)
Title.objects.bulk_create(packet_title)
ArtistTitle.objects.bulk_update(packet_artist_title)
SpotifyData.objects.bulk_update(packet_spotify_data)
except FileNotFoundError as e:
raise NoFilesException("No such file or directory") from e
end_time = timezone.now()
self.stdout.write(
self.style.SUCCESS(
f"Loading CSV took: {(end_time-start_time).total_seconds()} seconds."
)
)
I tried to use bulk this way but unfortunately it doesn't work

Getting Syntax error in this python code, how to eleminate?

////// section begins /* use this afl when buy is different from cover and sell is different from short / RequestTimedRefresh( 1, onlyvisible = False ) ; _SECTION_BEGIN( "Algoji.com intraday.afl" ); intra = ParamToggle( "Activate Intraday Mode", "NO|YES" ); per10 = Param( "Trade Entry From(HHMM)", 920, 900, 2300, 1 ); per11 = Param( "Trade Entry Upto(HHMM)", 1445, 900, 2300, 1 ); per12 = Param( "Trade Exit(HHMM)", 1515, 900, 2300, 100 ); pop= ParamToggle( "Percentage or Points", "Points|Percentage"); slp = Param( "StopLoss", 0, 0, 1000, 0.1 ); tsl= Param("Trail Stop", 0, 0, 1000, 0.1); tgtp = Param( "Target", 0, 0, 1000, 0.1 ); Col = ParamColor( "Color of Modified Signals", colorYellow ); delay= ParamToggle("Trade Intrabar?", "YES|Candle Completion"); dlong= ParamToggle("Disable Long?", "NO|YES"); dshort= ParamToggle("Disable Short?", "NO|YES"); if(dlong){Buy=Sell=0;} if(dshort){Short=Cover=0;} dd= DaysSince1900(); d=prof= 0; if(delay) {Buy=Ref(Buy,-1); Sell=Ref(Sell,-1); Short= Ref(Short,-1); Cover= Ref(Cover,-1);} qt= Param("Trade Quantity", 0, 0, 1000000) ; exposure= Param("Exposure", 0, 0, 1000000) ; if(exposure>0) qt= round(exposure/ValueWhen(Day()!=Ref(Day(),-1), C)); maxl= Param("Qty using SL (MaxLoss)",0,0,100000 ); if(maxl>0 AND !pop) qt= round(maxl/slp); if(maxl>0 AND pop) { basicprice= LastValue(ValueWhen(Day()!=Ref(Day(),-1), C)); sl= slpbasicprice/100; qt= round(maxl/sl); } intraex = intra AND (TimeNum() > per12 * 100); intraen = !intra OR ( TimeNum() <= per11 * 100 AND TimeNum() >= per10 * 100 ); Buy1 = Buy; Sell1 = Sell; Short1 = Short; Cover1 = Cover; Buy=Sell=Short=Cover=0; bflag = sflag = sp=bp = 0; slarr = tgtarr = qtarr= Null; for ( i = 10; i < BarCount; i++ ) { if ( ( Cover1[i] OR intraex[i]OR( H[i] > slarr[i-1] AND (sl>0 OR tsl>0) ) OR ( L[i] < tgtarr[i-1] AND tgt > 0 ) ) AND sflag ) { Cover[i] = 1; CoverPrice[i]= C[i]; sflag = 0; d= dd[i]; prof= sp-C[i]; } if ( ( Sell1[i] OR intraex[i] OR( L[i] < slarr[i-1] AND (sl>0 OR tsl>0) ) OR ( H[i] > tgtarr[i-1] AND tgt > 0 ) ) AND bflag ) { Sell[i] = 1; SellPrice[i]= C[i]; bflag = 0; d= dd[i]; prof= C[i]- bp; } if ( Buy1[i] AND intraen[i] AND bflag == 0 ) { Buy[i] = 1; bflag = 1; bp= C[i]; sl=slp; tgt= tgtp; if(pop) {sl= slpbp/100; tgt= tgtpbp/100;} if ( slp ) slarr[i] = bp-sl; if ( tgtp ) tgtarr[i] = bp+tgt; } if ( bflag AND Buy[i]==0 ) { slarr[i] = slarr[i-1]; tgtarr[i] = tgtarr[i-1]; if(tsl>0 AND pop) slarr[i] = Max(slarr[i-1], H[i](1-tsl/100)); if(tsl>0 AND !pop) slarr[i] = Max(slarr[i-1], H[i]-tsl); } if ( Short1[i] AND intraen[i] AND sflag == 0 ) { Short[i] = 1; sflag = 1; Sp= C[i]; sl= slp; tgt= tgtp; if(pop) {sl= slpSp/100; tgt= tgtpSp/100;} if ( slp ) slarr[i] = sp + sl; if ( tgtp ) tgtarr[i] = sp - tgt; } if ( sflag AND Short[i] == 0 ) { slarr[i] = slarr[i-1]; tgtarr[i] = tgtarr[i-1]; if(tsl>0 AND pop) slarr[i] = Min(slarr[i-1], L[i](1+tsl/100)); if(tsl>0 AND !pop) slarr[i] = Min(slarr[i-1], L[i]+tsl); } } Plot( slarr, "SL", Col, styleThick ); Plot( tgtarr, "TGT", Col, styleThick ); PlotShapes( IIf( Buy, shapeUpArrow, shapeNone ), Col, 0, H, Offset = 15 ); PlotShapes( IIf( Short, shapeDownArrow, shapeNone ), Col, 0, L, Offset = 15 ); PlotShapes( IIf( Cover, shapeStar, shapeNone ), Col, 0, H, Offset = -25 ); PlotShapes( IIf( Sell, shapeStar, shapeNone ), Col, 0, L, Offset = -25 ); sig = IIf( BarsSince( Buy ) < BarsSince( Short ), 1, 0 ); messageboard = ParamToggle( "Message Board", "Show|Hide", 1 ); if ( messageboard == 1 ) { GfxSelectFont( "Tahoma", 13, 100 ); GfxSetBkMode( 1 ); GfxSetTextColor( colorWhite ); GfxSelectSolidBrush( colorDarkTeal ); // this is the box background color pxHeight = Status( "pxchartheight" ) ; xx = Status( "pxchartwidth" ); Left = 1100; width = 310; x = 5; x2 = 310; y = pxHeight; GfxSelectPen( colorGreen, 1 ); // broader color GfxRoundRect( x, y - 160, x2, y , 7, 7 ) ; GfxTextOut( ""+WriteIf(intra, "Intraday Mode Activated", "Intraday Mode Not Activated" ), 13, y-160 ); GfxTextOut( ( "Current Qty "+qt ), 13, y-140 ); GfxTextOut( ( "Last" + " Signal came " + ( BarsSince( Buy OR Short ) ) * Interval() / 60 + " mins ago" ), 13, y - 120 ) ; // The text format location GfxTextOut( ( "" + WriteIf( sig == 1, "BUY # " + ValueWhen(Buy,C) , "SHORT # " + ValueWhen(Short,C) ) ), 13, y - 100 ); GfxTextOut( "Stop Loss : " + WriteIf(slp==0, "Not Activated", ""+slarr), 13, y - 80 ); GfxTextOut( "Target : " + WriteIf(tgtp==0, "Not Activated", ""+tgtarr), 13, y - 60 ); GfxTextOut( ( "Current P/L : " + WriteVal( IIf( sig == 1, (C-ValueWhen(Buy,C))*qt, (ValueWhen(Short,C)-C)*qt ), 2.2 ) ), 13, y-40 ); // GfxTextOut( ( "jhjh " ), 13, y-20 ); } //section begins for auto trade instr= ParamList("Instrument Name","EQ|FUTIDX|FUTSTK|OPTIDX|OPTSTK|FUTCOM"); stag= ParamStr("Strategy Tag", "STG1"); qty= NumToStr(qt[BarCount-1], 1.0, False) ; bp= sp= NumToStr(Close[BarCount-1],1.2, False); if(dlong){Buy=Sell=0;} if(dshort){Short=Cover=0;} if(delay) {Buy=Ref(Buy,-1); Sell=Ref(Sell,-1); Short= Ref(Short,-1); Cover= Ref(Cover,-1);} global algoji; algoji = Name() + NumToStr( Interval() / 60, 1.0, False ) ; procedure aStaticVarSet( SName, Svalue ) { global algoji; StaticVarSet( Sname + algoji, Svalue ); } function aStaticVarGet( SName ) { global algoji; Var = StaticVarGet( Sname + algoji ); if ( IsNull( Var = StaticVarGet( Sname + algoji ) ) ) Var = 0; return Var; } sym = Name(); //_TRACE("t"+t); Checkdt=Nz(aStaticVarGet("lastdt")); dt = LastValue( DateTime() ); Checkdtss=Nz(aStaticVarGet("lastdtss")); dtss = LastValue( DateTime() ); Checkdtc=Nz(aStaticVarGet("lastdtc")); dtc = LastValue( DateTime() ); Checkdts=Nz(aStaticVarGet("lastdts")); dts = LastValue( DateTime() ); RTBuy = LastValue( Buy) AND Checkdt != dt; RTShort = LastValue( Short) AND Checkdtss != dtss; RTCover = LastValue( Cover) AND Checkdtc != dtc; RTSell = LastValue( Sell) AND Checkdts != dts; if ( RTCover ) { aStaticVarSet("lastdtc",dtc ); StaticVarSet("counter", Nz(StaticVarGet("counter"))+1 ); _TRACE( "#"+Nz(StaticVarGet("counter"))+",SX,"+sym+",,," +bp +","+qty+","+instr+",,"); Algoji_Signal(NumToStr(Nz(StaticVarGet("counter")),0,False), "SX",sym,"M","",bp,qty,instr,stag); } if ( RTSell ) { aStaticVarSet("lastdts",dts ); StaticVarSet("counter", Nz(StaticVarGet("counter"))+1 ); _TRACE( "#"+Nz(StaticVarGet("counter"))+",LX,"+sym+",,," +sp +","+qty+",,,"); Algoji_Signal(NumToStr(Nz(StaticVarGet("counter")),0,False), "LX",sym,"M","",sp,qty,instr,stag); } if ( RTBuy ) { aStaticVarSet("lastdt",dt ); StaticVarSet("counter", Nz(StaticVarGet("counter"))+1 ); _TRACE( "#"+Nz(StaticVarGet("counter"))+",LE,"+sym+",,," +bp +","+qty+","+instr+",,"); Algoji_Signal(NumToStr(Nz(StaticVarGet("counter")),0,False), "LE",sym,"M","",bp,qty,instr,stag); } if ( RTShort ) { aStaticVarSet("lastdtss",dtss ); StaticVarSet("counter", Nz(StaticVarGet("counter"))+1 ); sp= NumToStr(Close[BarCount-1],1.2, False); _TRACE( "#"+Nz(StaticVarGet("counter"))+",SE,"+sym+",,," +sp +","+qty+","+instr+",,"); Algoji_Signal(NumToStr(Nz(StaticVarGet("counter")),0,False), "SE",sym,"M","",bp,qty,instr,stag); } Button = ParamToggle( "Enable Button Trading", "YES|NO" ); expiry= ParamStr("Expiry",""); strike= ParamStr("Strike",""); type= ParamStr("Option Type", ""); sym = Name()+ "|"+expiry+ "|" +strike+ "|" +type; function GetSecondNum() { Time = Now( 4 ); Seconds = int( Time % 100 ); Minutes = int( Time / 100 % 100 ); Hours = int( Time / 10000 % 100 ); SecondNum = int( Hours * 60 * 60 + Minutes * 60 + Seconds ); return SecondNum; } function PopupWindowEx( popupID, bodytext, captiontext, timeout, left, top ) { displayText = bodytext + captiontext; if ( ( StaticVarGetText( "prevPopup" + popupID ) != displayText) OR ( StaticVarGet( "prevPopupTime" + popupID ) < GetSecondNum() ) ) { StaticVarSetText( "prevPopup" + popupID, displayText); StaticVarSet( "prevPopupTime" + popupID, GetSecondNum() + timeout ); PopupWindow( bodytext, Captiontext + popupID, timeout, Left, top ); } } x1= Status( "pxchartleft" )+10; y1= Status( "pxcharttop" )+20; if ( Button == 0 ) { click = GetCursorMouseButtons() == 9; Px = GetCursorXPosition( 1 ); Py = GetCursorYPosition( 1 ); x2 = x1 + 60; y2 = y1 + 60; GfxSelectSolidBrush( ColorRGB( 0, 102, 0 ) ); //buy GfxSelectFont( "Tahoma", 13, 100 ); GfxSetBkMode( 1 ); GfxSetTextColor( colorWhite ); GfxRoundRect( x1, y1, x2, y2 , 7, 7 ) ; GfxTextOut( "LE", x1 + 20, y1 + 20 ); sx1 = x2; sy1 = y1; sx2 = sx1 + 60; sy2 = sy1 + 60; GfxSelectSolidBrush( ColorRGB( 255, 204, 204 ) );//sell GfxRoundRect( sx1, sy1, sx2, sy2 , 7, 7 ) ; GfxSetTextColor( ColorRGB( 153, 0, 0 ) ); GfxTextOut( "SE", sx1 + 20, sy1 + 20 ); ex1 = x1; ey1 = y1+60; ex2 = ex1 + 60; ey2 = ey1 + 60; GfxSelectSolidBrush( ColorRGB( 255, 204, 204 ) );//sell GfxRoundRect( ex1, ey1, ex2, ey2 , 7, 7 ) ; GfxSetTextColor( ColorRGB( 153, 0, 0 ) ); GfxTextOut( "LX", ex1 + 20, ey1 + 20 ); GfxSelectSolidBrush( ColorRGB( 0, 102, 0 ) );//sell GfxRoundRect( ex2, ey1, ex2+60, ey2 , 7, 7 ) ; GfxSetTextColor( colorWhite ); GfxTextOut( "SX", ex2 + 20, ey1 + 20 ); if ( px > x1 AND pxy1 AND py < y2 AND Click ) { _TRACE( "# ," + NumToStr(Nz(StaticVarGet("counter")),0,False) + ", BUY triggered from button, " ); AlertIf( 1, "SOUND C:\Windows\Media\tada.wav", "Audio alert", 1, 2, 1 ); StaticVarSet("counter", Nz(StaticVarGet("counter"))+1 ); PopupWindowEx( "ID:1", "BUY", "Buy Triggered from Button "+Name(), 1, -1, -1 ); AlgoJi_Signal(NumToStr(Nz(StaticVarGet("counter")),0,False), "LE",sym,"M","",sp,qty,instr,stag); } //https://algoji.com/ if ( px > sx1 AND pxsy1 AND py < sy2 AND Click ) { _TRACE( "# ," + NumToStr( DateTime(), formatDateTime ) + ", SHORT triggered from button, " ); AlertIf( 2, "SOUND C:\Windows\Media\tada.wav", "Audio alert", 2, 2, 1 ); StaticVarSet("counter", Nz(StaticVarGet("counter"))+1 ); PopupWindowEx( "ID:3", "SHORT", "Short Triggered from Button "+Name(), 1, -1, -1 ); AlgoJi_Signal(NumToStr(Nz(StaticVarGet("counter")),0,False), "SE",sym,"M","",sp,qty,instr,stag); } //https://algoji.com/ if ( px > ex1 AND pxey1 AND py<ey2 AND Click ) { _TRACE( "# ," + NumToStr( DateTime(), formatDateTime ) + ", SELL triggered from button, " ); AlertIf( 3, "SOUND C:\Windows\Media\tada.wav", "Audio alert", 2, 2, 1 ); StaticVarSet("counter", Nz(StaticVarGet("counter"))+1 ); PopupWindowEx( "ID:3", "SELL", "Sell Triggered from Button "+Name(), 1, -1, -1 ); AlgoJi_Signal(NumToStr(Nz(StaticVarGet("counter")),0,False), "LX",sym,"M","",sp,qty,instr,stag); } //https://algoji.com/ if ( px > ex2 AND px<(ex2+60) AND py>ey1 AND py < ey2 AND Click ) { _TRACE( "# ," + NumToStr( DateTime(), formatDateTime ) + ", Cover triggered from button, " ); AlertIf( 4, "SOUND C:\Windows\Media\tada.wav", "Audio alert", 2, 2, 1 ); StaticVarSet("counter", Nz(StaticVarGet("counter"))+1 ); PopupWindowEx( "ID:3", "Cover", "Cover Triggered from Button "+Name(), 1, -1, -1 ); AlgoJi_Signal(NumToStr(Nz(StaticVarGet("counter")),0,False), "SX",sym,"M","",sp,qty,instr,stag); } }
It is not a Python, it is AFL. It will be much easier if you post it as a code.
If you run it in AFL Formula Editor it will help you to debug it - simply click "Apply" and you will have all errors highlined.
I've found that you missed some "*" in your formula.

Using XmlWriter to create large document from LINQ to SQL / LINQPad throws Out of Memory Exception

I'm trying to export data in a LINQPad script and keep receiving Out of Memory exception. I feel like the script is doing all 'streamable' actions so not sure why I'm getting this.
The main loop of the code looks like the following. A few notes:
1) The first query returns around 60K rows profileDB.Profiles.Where(p => p.Group.gName == groupName).Select( d => d.pAuthID )
2) The second query for each pAuthID returns rows in a the database where one field is a Xml blob of data stored in a string field. It is not that big...< 500K for sure. Each pAuthID row could have as many as 50 rows of FolderItems. The query is profileDB.FolderItems.Where(f => f.Profile.pAuthID == p && ( folderTypes[0] == "*" || folderTypes.Contains(f.fiEntryType) ) ).OrderBy(f => f.fiEntryDate)
3) I only write a single line to the result pane when the processing starts.
4) The script runs for a long time, throwing exception when the output file is around 600-700MB. Huge I know, but it is a requirement that we dump out all the data into Xml.
5) The WriteFolderItems function/loop will be pasted below the main loop.
6) I call XmlWriter.Flush after each xDataDef element.
using (var xw = XmlWriter.Create(fileName, new XmlWriterSettings { Indent = false } ) )
{
xw.WriteStartElement( "xDataDefs" );
foreach( var p in profileDB.Profiles.Where(p => p.Group.gName == groupName).Select( d => d.pAuthID ) )
{
if ( totalRows == 0 ) // first one...
{
string.Format( "Writing results to {0}...", fileName ).Dump( "Progress" );
}
totalRows++;
var folderItems = profileDB.FolderItems.Where(f => f.Profile.pAuthID == p && ( folderTypes[0] == "*" || folderTypes.Contains(f.fiEntryType) ) ).OrderBy(f => f.fiEntryDate);
if ( folderItems.Any() )
{
xw.WriteStartElement("xDataDef");
xw.WriteAttributeString("id-auth", p);
xw.WriteStartElement("FolderItems");
WriteFolderItems(profileDB, datalockerConnectionString, xw, folderItems, documentsDirectory, calcDocumentFolder, exportFileData);
xw.WriteEndElement();
xw.WriteEndElement();
xw.Flush();
}
}
xw.WriteEndElement();
}
WriteFolderItems has looping code as well that looks like the following. A few notes:
1) I'd expect the foreach( var f in folderItems ) to be streaming
2) For some of the FolderItem rows that are Xml blobs of cached documents, I need to run ~ 1-5 queries against the database to get some additional information to stick into the Xml export: var docInfo = profileDB.Documents.Where( d => d.docfiKey == f.fiKey && d.docFilename == fileName ).FirstOrDefault();
3) I call XmlWriter.Flush after each FolderItem row.
public void WriteFolderItems( BTR.Evolution.Data.DataContexts.Legacy.xDS.DataContext profileDB, string datalockerConnectionString, XmlWriter xw, IEnumerable<BTR.Evolution.Data.DataContexts.Legacy.xDS.FolderItem> folderItems, string documentsOutputDirectory, string calcDocumentFolder, bool exportFileData )
{
foreach( var f in folderItems )
{
// The Xml blob string
var calculation = XElement.Parse( f.fiItem );
// If it contains 'cached-document' elements, need to download the actual document from DataLocker database
foreach( var document in calculation.Elements( "Data" ).Elements( "TabDef" ).Elements( "cache-documents" ).Elements( "cached-document" ) )
{
var fileName = (string)document.Attribute( "name" );
// Get author/token to be used during import
var docInfo = profileDB.Documents.Where( d => d.docfiKey == f.fiKey && d.docFilename == fileName ).FirstOrDefault();
if ( docInfo != null )
{
document.Add( new XElement( "author", docInfo.docUploadAuthID ) );
document.Add( new XElement( "token", docInfo.docDataLockerToken ) );
}
// Export associated document from DataLocker connection...XmlWriter is not affected, simply saves document to local hard drive
if ( exportFileData && DataLockerExtensions.ByConnection( datalockerConnectionString ).Exists( calcDocumentFolder, (string)document.Attribute( "name" ), null ) )
{
using ( var fs = new FileStream( Path.Combine( documentsOutputDirectory, fileName.Replace( "/", "__" ) ), FileMode.Create ) )
{
string contentType;
using ( var ds = DataLockerExtensions.ByConnection( datalockerConnectionString ).Get( calcDocumentFolder, (string)document.Attribute( "name" ), null, out contentType ) )
{
ds.CopyTo( fs );
}
}
}
}
// Write the calculation to the XwlWriter
xw.WriteStartElement( "FolderItem" );
xw.WriteElementString( "Key", f.fiKey.ToString() );
xw.WriteElementString( "EntryDate", XmlConvert.ToString( f.fiEntryDate.Value, XmlDateTimeSerializationMode.Local ) );
xw.WriteElementString( "ItemType", f.fiEntryType );
xw.WriteElementString( "Author", f.fiAuthor );
xw.WriteElementString( "Comment", f.fiComment );
xw.WriteStartElement( "Item" );
calculation.WriteTo( xw );
xw.WriteEndElement();
xw.WriteEndElement();
xw.Flush();
}
}
Make sure you disable Change Tracking, or the EF or L2S Change Tracker will retain references to each of the loaded entities.

Run connectedComponents of GraphX with the same data and code twice,the result is different

When I run connectedComponents of GraphX with the same data and code twice,the result is different,why?
Here's my code:
val edgeDF = sql("select * from fkdm.fkdm_fk_base_sna_bm_edge_s_d").rdd
val edgePair = edgeDF.map{ line =>
val v1 = line.getString(0)
val v2 = line.getString(1)
(v1,v2)
}
val vertices = edgePair.map(line => line._1).union(edgePair.map(line =>
line._2)).distinct()
val verticesUniqueId = vertices.zipWithUniqueId()
val edge_numberic = edgePair.join(verticesUniqueId).map(line =>
(line._2._1,line._2._2)).join(verticesUniqueId).map(line =>
(line._2._1,line._2._2))
val vertice_rdd = verticesUniqueId.map{line =>
(line._2.toLong,line._1)
}
val edge_rdd = edge_numberic.map{line =>
Edge(line._1,line._2,1)
}
val graph = Graph(vertice_rdd,edge_rdd)
val cc = graph.connectedComponents()
val ccDF = cc.vertices.map(x =>
ComponentsGraph(x._1.toString,x._2.toString)).toDF()
ccDF.createOrReplaceTempView("temp_ccDF")
sql("drop table if exists buming.buming_sna_vertices_groupnum ")
sql("CREATE TABLE buming.buming_sna_vertices_groupnum AS SELECT * FROM
temp_ccDF")
I found the number of vertices in the largest group is diffent.

how to program this indicator script to work with stocks and not just futures

i found this.. posted it in the Amibroker editor.. saved.. found out after looking into it further.. that it will only work on futures that are listed in the if statements within the code.. i'd like to see this for stocks.. any ideas..
// ACD Plot
// LSMA is Linreg
// ACD.afl
// v 1.2 9/13/2004
SetChartBkColor(16);
Per = Param("Periods",13);
Per2 = Param("Periods 2",34);
LSMAPer = Param("LMSA Period",25);
Offset = Param("A Level",1);
ACDFlag = 0;
IntervalFlag = IIf(Interval(format=0)==300,1,0);
strInterval = Interval(format=2);
strWeekday = StrMid("SunMonTueWedThuFriSat", SelectedValue(DayOfWeek())*3,3);
if( StrFind( Name(), "YM" ) )
{
ACDOffset = 10;
ACDFlag = 1;
ACDTime = 94500;
}
if( StrFind( Name(), "ER" ) )
{
ACDOffset = 0.5;
ACDFlag = 1;
ACDTime = 93500;
}
if( StrFind( Name(), "ES" ) )
{
ACDOffset = 2;
ACDFlag = 1;
ACDTime = 94500;
}
if( StrFind( Name(), "NQ" ) )
{
ACDOffset = 3;
ACDFlag = 1;
ACDTime = 94500;
}
if( StrFind( Name(), "ZB" ) )
{
ACDOffset = 0.15;
ACDFlag = 1;
ACDTime = 83000;
}
if( StrFind( Name(), "ZN" ) )
{
ACDOffset = 0.15;
ACDFlag = 1;
ACDTime = 83000;
}
GraphXSpace = 1;
Shift = 2;
// calculate the pivot range
PDH = TimeFrameGetPrice( "H", inDaily, -1 ); // gives previous Day High when working on intraday data
PDL = TimeFrameGetPrice( "L", inDaily, -1 );
PDC = TimeFrameGetPrice( "C", inDaily, -1 );
PP = (PDH+PDL+PDC)/3;
DIFF = abs((PDH+PDL)/2 - PP);
PRHi = PP + DIFF;
PRLo = PP - DIFF;
EMA1 = EMA(Avg,Per);
EMA2 = EMA(Avg,Per2);
LSMA = LinearReg(Avg, LSMAPer);
Plot(C, "Close",colorWhite,styleCandle);
if (ACDFlag AND IntervalFlag) {
ORHigh= ValueWhen(TimeNum()<ACDTime,HighestSince(DateNum()>Ref(DateNum(),-1),High));
ORLow = ValueWhen(TimeNum()<ACDTime,LowestSince(DateNum()>Ref(DateNum(),-1), Low));
Plot(PRHi,"PRHigh",colorWhite,styleDots+styleNoLine+styleNoLabel);
Plot(PRLo,"PRLow",colorWhite,styleDots+styleNoLine+styleNoLabel);
Plot(ORHigh,"ORHigh",colorBlue,style=styleStaircase+styleDots+styleNoLine+styleNoLabel);
Plot(ORLow,"ORLow",colorBlue,style=styleStaircase+styleDots+styleNoLine+styleNoLabel);
Plot(ORHigh+ACDOffset,"AUp",colorYellow,style=styleStaircase+styleDots+styleNoLine);
Plot(ORLow-ACDOffset,"ADn",colorYellow,style=styleStaircase+styleDots+styleNoLine);
// Plot(LSMA, "LSMA", colorYellow,style=styleThick);
}
Title=Name()+" ["+strInterval+"] "+ strWeekday + " " +Date()+ " Close: "
+WriteVal(C,format=1.2) +" "+WriteVal(per,format=1.0)+"-Per MA: "
+WriteVal(EMA1,format=1.2)+" " + WriteVal(per2,format=1.0)+"-Per MA: "
+WriteVal(EMA2,format=1.2) + " PR High: "+WriteVal(PRHi,format=1.2) + " PR Low: "
+WriteVal(PRLo,format=1.2);
this is the response i got in Amibroker forum.. thought i would share the answer..
This is both indicator and an exploration (however – this code is using very old approach, now it’s much more convenient to use PLOT or ADDCOLUMN functions instead of this obsolete coding style)
There are just some errors in the formula, as it uses assignment instead of equality check, so you need to replace:
HiLo=IIf(HLv=-1,
with
HiLo=IIf(HLv == -1,
the same with - HiLoInvert=IIf(HLv=-1,
This is because == (double =) is an operator used for equality check.

Resources