RocksDB JNI Slow Read Performance - database

I am using RocksDB JNI and I found that reads are getting exponentially slow for this program
Initially, it's in an acceptable range. But when the program runs so that the total records reach 1 million then the read time logger prints are showing around 200-300 ms. Getting still worse, as the program runs. Am I using the JNI wrong?
long n = 0;
int counter = 0;
try(
final Options options = new Options()
.setCreateIfMissing(true)
.setComparator(new Comparator(new ComparatorOptions()){
#Override
public String name() {
return "default";
}
#Override
public int compare(final Slice a, final Slice b) {
long x = ByteBuffer.wrap(a.data()).getLong();
long y = ByteBuffer.wrap(b.data()).getLong();
return (x < y) ? -1 : ((x == y) ? 0 : 1);
}
})
.setWriteBufferSize(64 * SizeUnit.MB)
.setMaxWriteBufferNumber(6)
.setMinWriteBufferNumberToMerge(2);
final RocksDB db = RocksDB.open(options, "/PathToDB/")){
boolean loop = true;
while(loop) {
if(n == Long.MAX_VALUE){
loop = false;
}
for (int j=0;j<4;j++){
try(WriteBatch writeBatch = new WriteBatch()) {
for (int i = 0; i < 200; i++) {
String urlStr = "dummy"+counter;
counter++;
Long score = getScore(urlStr);
ByteBuffer buf = ByteBuffer.allocate(Long.BYTES);
buf.putLong(score);
writeBatch.put( buf.array() , urlStr.getBytes(UTF_8));
}
long st = System.currentTimeMillis();
db.write(new WriteOptions(), writeBatch);
long et = System.currentTimeMillis();
logger.logMessage(Test.class.getName(), Level.INFO, "RocksDB write of 200 URLs successful. Time taken - {0}", new Object[]{ et-st});
} catch (RocksDBException ex) {
}
}
byte[] firstKey = null, lastKey = null;
int readC = 0;
long st = System.currentTimeMillis();
final RocksIterator it = db.newIterator();
it.seekToFirst();
while(it.isValid() && readC < 50){
lastKey = it.key();
if(firstKey == null){
firstKey = lastKey;
}
it.next();
readC++;
}
long et = System.currentTimeMillis();
logger.logMessage(Test.class.getName(), Level.INFO, "RocksDB read of 50 URLs successful. Time taken - {0}", new Object[]{ et-st});
if(lastKey != null){
db.deleteRange(firstKey, lastKey);
}
n++;
}
}catch (Exception e){
logger.logMessage(Level.SEVERE, Test.class.getName(), e);
}

Related

Optimizing Replacing Bytes of Some File in Java 8

I have this method in Java and I want to improve it.
The method is used to replace some part (at the beginning, in the middle or at the end) of some File with the new bytes (the selected part can be replaced by less or more bytes).
The selection is done, by position(start) and quantity.
I can't to use external libraries (guava, or some other).
Here my old code:
public static void replaceBytesFile(RandomAccessFile rafTarget,
byte[] replacers, int start, int quantity) {
//replaces exact amount of bytes of a file starting at a specified position
RandomAccessFile rafTemp = null;
//Ini Select a Random NonExistent File
File userDirectory = new File(System.getProperty("user.dir"));
File temporalFile;
boolean existsTemporalFile = false;
String temporalFilename = "";
while (!existsTemporalFile) {
temporalFilename = "File_" + Double.toString(100000 * Math.random()) + ".tmp";
temporalFilename = userDirectory + MethodGen.FS + temporalFilename;
temporalFile = new File(temporalFilename);
if (!temporalFile.exists()) {
existsTemporalFile = true;
}
}
//End Select a Random NonExistent File
try {
rafTemp = new RandomAccessFile(temporalFilename, "rw");
int workBufferSize = 65536;
//Ini Copy first (Start - 1) MethodBytes
int step = workBufferSize;
int countPosition = 0;
while (countPosition < start) {
rafTarget.seek(countPosition);
rafTemp.seek(countPosition);
if ((start - countPosition) < step) {
step = start - countPosition;
}
byte[] WorkBuffer = new byte[step];
rafTarget.read(WorkBuffer);
rafTemp.write(WorkBuffer);
countPosition += step;
}
//End Copy first (start - 1) MethodBytes
rafTemp.write(replacers);
rafTarget.seek(start + quantity);
int end = (int) rafTarget.length();
//Ini Copy last MethodBytes
step = workBufferSize;
countPosition = start + quantity;
while (countPosition < end) {
rafTarget.seek(countPosition);
rafTemp.seek(countPosition - quantity + replacers.length);
if ((end - countPosition) <= step) {
step = end - countPosition;
}
byte[] WorkBuffer = new byte[step];
rafTarget.read(WorkBuffer);
rafTemp.write(WorkBuffer);
countPosition += step;
}
//End Copy last MethodBytes
rafTarget.setLength(0);
step = workBufferSize;
countPosition = 0;
end = (int) rafTemp.length();
//Ini Copy all MethodBytes to original
while (countPosition < end) {
rafTemp.seek(countPosition);
rafTarget.seek(countPosition);
if ((end - countPosition) <= step) {
step = end - countPosition;
}
byte[] WorkBuffer = new byte[step];
rafTemp.read(WorkBuffer);
rafTarget.write(WorkBuffer);
countPosition += step;
}
//End Copy all MethodBytes to original
rafTemp.close();
temporalFile = new File(temporalFilename);
temporalFile.delete();
} catch (IOException ioe) {
System.out.println(ioe.toString());
} finally {
try {
if (rafTemp != null) {
rafTemp.close();
}
} catch (IOException e) {
}
}
}
I'm copying manually in from original file to temporal file where the changes are performed, later ,
My code is working, but I want to know some best alternative in Java 8 (preferred).
Now How is test?
public static void main(String[] args) {
String originalFilename = "OriginalTraveling.txt";
String copiedFilename = "TravelingToBeChanged.txt";
Path copiedPath = Paths.get(copiedFilename);
Path originalPath = new File(originalFilename).toPath();
System.out.println("filename:" + originalFilename);
String contet = "I want to travel to my Country.";
try {
RandomAccessFile raf = new RandomAccessFile(originalFilename, "rw");
putBytesFile(raf, contet.getBytes(), 0);
Files.copy(originalPath, copiedPath, StandardCopyOption.REPLACE_EXISTING);
}
catch (IOException e) {
System.out.println("Exception caught " + e.toString());
}
try {
RandomAccessFile raf = new RandomAccessFile(copiedFilename, "rw");
String toBeChanged = "my Country.";
String toBeInserted = "India, China, Europe, Latin America, Australia.";
int position = contet.indexOf(toBeChanged);
replaceBytesFile(raf, toBeInserted.getBytes(), position, toBeChanged.length());
}
catch (IOException e) {
System.out.println("Exception caught " + e.toString());
}
try {
RandomAccessFile raf = new RandomAccessFile(copiedFilename, "rw");
String replacedContent = new String(getBytesFile(raf, 0, (int) raf.length()));
String toBeChanged = "Latin America";
String toBeInserted = "Colombia";
int position = replacedContent.indexOf(toBeChanged);
replaceBytesFile(raf, toBeInserted.getBytes(), position, toBeChanged.length());
} catch (IOException e) {
System.out.println("Exception caught " + e.toString());
}
}
Method to put Bytes!
public static void putBytesFile(RandomAccessFile RAFTarget, byte[] content, int position) {
int size = content.length;
try {
long oldPosition = RAFTarget.getFilePointer();
if (!((position < 0) || !(size > 0))) {
RAFTarget.seek(position);
RAFTarget.write(content);
RAFTarget.seek(oldPosition);
}
} catch (java.io.IOException e) {
System.out.println(e.toString());
}
}
Method Get Files!
public static byte[] getBytesFile(RandomAccessFile RAFSource, int position, int quantity) {
byte[] content = null;
try {
long oldPosition = RAFSource.getFilePointer();
if ((position < 0) || !(quantity > 0)) {
return (content);
} else {
if (RAFSource.length() < (position + quantity)) {
quantity = (int) RAFSource.length() - position;
}
RAFSource.seek(position);
content = new byte[quantity];
RAFSource.read(content);
RAFSource.seek(oldPosition);
}
} catch (java.io.IOException e) {
System.out.println(e.toString());
}
return content;
}
Content of OriginalTraveling.txt
I want to travel to my Country.
Content of TravelingToBeChanged.txt
I want to travel to India, China, Europe, Latin America, Australia.
Finally the Content of TravelingToBeChanged.txt
I want to travel to India, China, Europe, Colombia, Australia.
If it can be noticed, they are NOT changed by the same number of bytes.
Do you know some alternative to replace contents of File?
Even for ancient code, this looks unnecessary complicated.
E.g. instead of
//Ini Select a Random NonExistent File
File userDirectory = new File(System.getProperty("user.dir"));
File temporalFile;
boolean existsTemporalFile = false;
String temporalFilename = "";
while (!existsTemporalFile) {
temporalFilename = "File_" + Double.toString(100000 * Math.random()) + ".tmp";
temporalFilename = userDirectory + MethodGen.FS + temporalFilename;
temporalFile = new File(temporalFilename);
if (!temporalFile.exists()) {
existsTemporalFile = true;
}
}
just use
File temporalFile = File.createTempFile("File_", ".tmp", userDirectory);
See createTempFile
Further, instead of
int step = workBufferSize;
int countPosition = 0;
while (countPosition < start) {
rafTarget.seek(countPosition);
rafTemp.seek(countPosition);
if ((start - countPosition) < step) {
step = start - countPosition;
}
byte[] WorkBuffer = new byte[step];
rafTarget.read(WorkBuffer);
rafTemp.write(WorkBuffer);
countPosition += step;
}
Use
for(int step=workBufferSize, countPosition=0; countPosition < start; countPosition += step){
rafTarget.seek(countPosition);
rafTemp.seek(countPosition);
if ((start - countPosition) < step) {
step = start - countPosition;
}
byte[] WorkBuffer = new byte[step];
rafTarget.read(WorkBuffer);
rafTemp.write(WorkBuffer);
}
As you clearly have an initial statement, a condition and an increment operation, in other words, a typical for loop. The same applies to the other two while loops.
However, with newer APIs, things are much simpler anyway:
// consider using long for position and Path for the file, unless
// the RandomAccessFile is really needed for other purposes
public static void replaceBytesFile(RandomAccessFile rafTarget,
byte[] replacers, int start, int quantity) throws IOException {
// no need to force a particular directory for the temp file
Path tmp = Files.createTempFile("File_", ".tmp");
// use import static java.nio.file.StandardOpenOption.*;
// try( ... ) closes automatically, perfect for a temp file with DELETE_ON_CLOSE
try(FileChannel tmpCh = FileChannel.open(tmp, READ, WRITE, DELETE_ON_CLOSE)) {
// closing the target channel would also close rafTarget RandomAccessFile
FileChannel target = rafTarget.getChannel();
// just keep the data before start position, only copy remainder
long retainStart = start + (long)quantity, toCopy = target.size() - retainStart;
target.transferTo(retainStart, toCopy, tmpCh);
// write the replacement
target.write(ByteBuffer.wrap(replacers), start);
// copy back the remainder, to the new position
tmpCh.position(0);
target.transferFrom(tmpCh, start + (long)replacers.length, toCopy);
// adapt the length if necessary
target.truncate(start + toCopy + replacers.length);
}
}

Putting array with unknown variables into another array

The purpose of this code is is to define the root of the sum of the squares.
I cant figure out how to put i into j. Please help.
namespace ConsoleApplication1
{
class Program
{
static void Main(string[] args)
{
int input, som, i=0;
int j = 0;
double answer;
Boolean gaDoor= true;
int [] array = new int [24];
while (gaDoor)
{
Console.Write("Specify a positive integer");
input = Convert.ToInt32(Console.ReadLine());
if (input == -1)
{
gaDoor = false;
}
else
{
if (input >= 0)
{
array[i] = input;
i++;
}
else
{
Console.WriteLine("Specify a positive integer ");
}
}
}
while (j<i)
{
sum = array [j] ^ 2;
answer = Math.Sqrt(sum);
Console.Write(answer);
}
Console.ReadKey();
}
}
}
using System;
namespace Test
{
class MainClass
{
public static void Main (string[] args)
{
int[] invoer = new int[24];
double[] resultaat = new double[24];
double totaal = 0;
double wortel = 0;
int commando = 0;
int teller = -1;
try {
// Keep going until a negative integer is entered (or a 0)
while ((commando = Convert.ToInt32 (Console.ReadLine ())) > 0) {
teller++;
invoer [teller] = commando;
}
} catch (FormatException) {
// Not a number at all.
}
teller = -1;
foreach (int i in invoer) {
teller++;
resultaat [teller] = Math.Pow (invoer [teller], 2);
totaal += resultaat [teller];
if (invoer [teller] > 0) {
Console.WriteLine ("Invoer: {0}, Resultaat: {1}", invoer [teller], resultaat [teller]);
}
}
wortel = Math.Sqrt (totaal);
Console.WriteLine ("Totaal: {0}, Wortel: {1}", totaal, wortel);
}
}
}

How to accelerate loading resource file in J2ME

I'm writing a J2ME application using J2ME Wireless Toolkit 2.2
I have the following code:
public class BusReader
{
private String[] fileNames;
private final String allFilesInfoFile = "files_in_dir";
public BusReader ()
{
fileNames = getFileNames ();
String busNo = getBusNo ("BusNo1p.bin");
}
public String[] getAllBusFiles ()
{
return fileNames;
}
public String getBusNo (String fileName)
{
String[] fileLines = loadResourceFile (fileName);
int linesCount = fileLines.length;
for (int i=0;i<linesCount;++i)
if (fileLines[i].equals ("[BusNo]") && i < linesCount-1)
return fileLines[i+1];
return null;
}
public String getDefaultDirection (String fileName)
{
String[] fileLines = loadResourceFile (fileName);
int linesCount = fileLines.length;
for (int i=0;i<linesCount;++i)
if (fileLines[i].equals ("[BusDirection]") && i < linesCount-1)
return fileLines[i+1];
return null;
}
private String[] getFileNames ()
{
return loadResourceFile (allFilesInfoFile);
}
private String[] loadResourceFile (String fileName)
{
String content = "";
try
{
Reader in = new InputStreamReader(this.getClass().getResourceAsStream(fileName), "iso-8859-2");
StringBuffer temp = new StringBuffer(1024);
char[] buffer = new char[1024];
int read;
while ((read=in.read(buffer, 0, buffer.length)) != -1)
temp.append(buffer, 0, read);
content = temp.toString();
} catch (IOException e) {
return null;
}
int len = content.length ();
if (content.charAt (len-1) == '\n' && content.charAt (len-2) == '\r')
{
String newContent = "";
for (int i=0;i<len-2;++i)
newContent += content.charAt (i);
content = newContent;
}
String[] fileLines = TString.Split ("\r\n", new TString(content));
for (int i=0;i<fileLines.length; ++i)
{
fileLines[i] = fileLines[i].trim ();
if (fileLines[i].length () == 0)
fileLines[i] = "";
}
return fileLines;
}
}
All works correctly, but the problem is when I'm trying to copy my application into mobile phone. On my mobile phone when I open the application before application show I must wait 35 seconds. This is because, constructor executes the function twice:
loadResourceFile (String fileName)
which loads the resource file. Files size which function is loading are: 1.22KB and 29KB.
The question is: How to accelerate loading function (loadResourceFile)?
I tried to create java class files as resource data but it exceeded java memory limit. I changed arrays String[][][][] to String[][][] and it was loading on my mobile phone in 15 seconds. I thought when I will load data as a resource it will work faster. My mobile phone: Nokia 3110c
I found it.
It was following lines (it takes 35 seconds):
if (content.charAt (len-1) == '\n' && content.charAt (len-2) == '\r')
{
String newContent = "";
for (int i=0;i<len-2;++i)
newContent += content.charAt (i);
content = newContent;
}

Null Pointer Exception using Object Arrays

I am planning to make an airline system. I have initialized the array using initSeats but it still throws back the NPE error. It happens when i call the seatChecker() from bookMenu.
public void initSeats(){
for(int b = 0; b < seatList.length; b++)
{
initC.setName("null");
initC.setEmail("null");
initC.setCreditNo(0);
initC.setAddress("null");
initC.setPassportNo("null");
seatList[b] = new Seat('A', 0, "null", 0.0, "Available", initC);
}
for(int d = 0; d <= 24; d++)
{
seatList[d].setSeatLetter('A');
seatList[d].setSeatNo(d);
}
for(int n = 25; n <= 48; n++)
{
seatList[n].setSeatLetter('B');
seatList[n].setSeatNo(n);
}
for(int m = 49; m <= 72; m++)
{
seatList[m].setSeatLetter('C');
seatList[m].setSeatNo(m);
}
for(int t = 73; t <= 96; t++)
{
seatList[t].setSeatLetter('D');
seatList[t].setSeatNo(t);
}
for(int q = 97; q <= 120; q++)
{
seatList[q].setSeatLetter('E');
seatList[q].setSeatNo(q);
}
for(int v = 121; v < 144; v++)
{
seatList[v].setSeatLetter('F');
seatList[v].setSeatNo(v);
}
for(int x = 0; x <= 48; x++)
{
seatList[x].setSection("Front");
seatList[x].setPrice(500);
}
for(int j = 49; j <= 96; j++)
{
seatList[j].setSection("Middle");
seatList[j].setPrice(250);
}
for(int u = 97; u < 144; u++)
{
seatList[u].setSection("Back");
seatList[u].setPrice(100);
}
}
public void seatChecker(int index)
{
String status = seatList[index].getStatus();
if(status.equalsIgnoreCase("Available")){
System.out.println("Seat is Available.");
}else{
System.out.println("Seat is not Available. Please Pick Another Seat.");
bookMenu();
}
}
public void bookMenu()
{
int choice1 = 0;
int index;
System.out.println("Where do you want to be seated?");
System.out.println("[1] Front");
System.out.println("[2] Middle");
System.out.println("[3] Back");
choice1 = sc.nextInt();
sc.nextLine();
if(choice1 == 1){
System.out.print("Choose a seat number (0 - 48): ");
index = sc.nextInt();
sc.nextLine();
seatChecker(index);
}else if(choice1 == 2){
System.out.println("Choose a seat number (49 - 96): ");
index = sc.nextInt();
sc.nextLine();
seatChecker(index);
}else if(choice1 == 3){
System.out.println("Choose a seat number (97 - 144): ");
index = sc.nextInt();
sc.nextLine();
seatChecker(index);
}else
{
System.out.println("Invalid Choice. Going back to Menu.");
MainMenu();
}
}
Null Pointer Exception Code
Exception in thread "main" java.lang.NullPointerException
at pkg.Airlines.AirlineUI.seatChecker(AirlineUI.java:132)
Seat Class
public class Seat{
private char seatLetter;
private int seatNo;
private String section;
private double price;
private String status;
private Customer customerDetails;
public Seat(char seatLetter, int seatNo, String section, double price, String status, Customer details)
{
this.seatLetter = seatLetter;
this.seatNo = seatNo;
this.section = section;
this.price = price;
this.status = status;
this.customerDetails = details;
}
public Customer getCustomerDetails() {
return customerDetails;
}
public void setCustomerDetails(Customer customerDetails) {
this.customerDetails = customerDetails;
}
public char getSeatLetter() {
return seatLetter;
}
public void setSeatLetter(char seatLetter) {
this.seatLetter = seatLetter;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public double getPrice() {
return price;
}
public void setPrice(double price) {
this.price = price;
}
public int getSeatNo() {
return seatNo;
}
public void setSeatNo(int seatNo) {
this.seatNo = seatNo;
}
public String getSection() {
return section;
}
public void setSection(String section) {
this.section = section;
}
}
Probably the problems are this two line :
String status = seatList[index].getStatus();
if(status.equalsIgnoreCase("Available"))
First thing could be seatList[index] is not initialized . Once you declare an array of references as :
Seat[] array = new Seat[10];
The array contains 10 null references for Seat object . You need to instantiate them before using them :
Seat[0] = new Seat();
Second potential problem will be, this check :
if(status.equalsIgnoreCase("Available"))
Replace it to :
if("Available".equalsIgnoreCase(status))
to avoid any NullPointerException in case status is null.
P.S. Please show us the Seat class to understand your problem better.
Well is quite simple resolve a Null Pointer Exception.
Probably in one of the index of seatList there isn't a value.

How to do a database lock in AppEngine (GAE)?

In GAE, I've got a table full of "one offs" -- things like "last-used sequence number" and the like that don't really fall into other tables. It's a simple String-key with String-value pair.
I've got some code to grab a named integer and increment it, like so:
#PersistenceCapable(detachable="true")
public class OneOff
{
#PrimaryKey
#Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
private Key key;
#Persistent
private String dataKey;
#Persistent
private String value;
public OneOff(String kk, String vv)
{
this.dataKey = kk;
this.value = vv;
}
public static OneOff persistOneOff(String kk, String vv)
{
OneOff oneoff= new OneOff(kk, vv);
PersistenceManager pm = PMF.get().getPersistenceManager();
try
{
pm.makePersistent(oneoff);
}
finally
{
pm.close();
}
return oneoff;
}
// snip...
#SuppressWarnings("unchecked")
synchronized
public static int getIntValueForKeyAndIncrement(String kk, int deFltValue)
{
int result = 0;
OneOff oneOff = null;
PersistenceManager pm = PMF.get().getPersistenceManager();
Query query = pm.newQuery(OneOff.class);
query.setFilter("dataKey == kkParam");
query.declareParameters("String kkParam");
List<OneOff> oneOffs = (List<OneOff>) query.execute(kk);
int count = oneOffs.size();
if (count == 1)
{
oneOff = oneOffs.get(0);
result = Integer.parseInt(oneOff.value);
}
else if (count == 0)
{
oneOff = new OneOff(kk, "default");
result = deFltValue;
}
else
{
// Log WTF error.
}
// update object in DB.
oneOff.value = "" + (result+1);
try
{
pm.makePersistent(oneOff);
}
finally
{
pm.close();
}
return result;
}
// etc...
However, when I make these calls:
int val1 = OneOff.getIntValueForKeyAndIncrement("someKey", 100);
int val2 = OneOff.getIntValueForKeyAndIncrement("someKey", 100);
int val3 = OneOff.getIntValueForKeyAndIncrement("someKey", 100);
Sometimes I get the desired increment and sometimes I get the same value. It appears that my DB access is running asynchronously, when I'd like to lock the DB for this particular transaction.
I thought that
synchronized
public static
was supposed to do that for me, but apparently not (probably due to multiple instances running!)
At any rate -- how do I do the thing that I want? (I want to lock my DB while I get & update this value, to make the whole thing concurrency-safe.)
Thanks!
== EDIT ==
I have accepted Robert's as the correct answer, since transactions were, indeed, what I wanted. However, for completeness, I have added my updated code below. I think it's correct, although I'm not sure about the if(oneOff==null) clause (the try-catch bit.)
public static int getIntValueForKeyAndIncrement(String kk, int defltValue)
{
int result = 0;
Entity oneOff = null;
int retries = 3;
// Using Datastore Transactions
DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
while (true)
{
com.google.appengine.api.datastore.Transaction txn = datastore.beginTransaction();
try
{
Key oneOffKey = KeyFactory.createKey("OneOff", kk);
oneOff = datastore.get (oneOffKey);
result = Integer.parseInt((String) oneOff.getProperty("value"));
oneOff.setProperty("value", "" + (result+1));
datastore.put(oneOff);
txn.commit();
break;
}
catch (EntityNotFoundException ex)
{
result = defltValue;
}
catch (ConcurrentModificationException ex)
{
if (--retries < 0)
{
throw ex;
}
}
if (oneOff == null)
{
try
{
Key oneOffKey = KeyFactory.createKey("OneOff", kk);
oneOff = new Entity(oneOffKey);
oneOff.setProperty("value", "" + (defltValue+1));
datastore.put(txn, oneOff);
datastore.put(oneOff);
txn.commit();
break;
}
finally
{
if (txn.isActive())
{
txn.rollback();
}
}
}
else
{
if (txn.isActive())
{
txn.rollback();
}
}
}
return result;
}
You should be updating your values inside a transaction. App Engine's transactions will prevent two updates from overwriting each other as long as your read and write are within a single transaction. Be sure to pay attention to the discussion about entity groups.

Resources