Room can't build database in Android Studio I can't figure out what's wrong? - database

I use a Room database class on Android Studio.
I have the following entities:
User, Address, Geo, Company, Album, Photo, AlbumPhotoCrossRef (there's many-to-many relationship between Album and Photo). And I added Word too just to test.
The code for these entities are added below.
When I use only Word (comment out other classes in the entities = part, it works, I can see the table via DatabaseInspector.
But when include all entities, the database don't even open, I can't even see name of the database on DatabaseInspector.
And I get the following error:
E/AndroidRuntime: FATAL EXCEPTION: pool-2-thread-1
Process: com.example.lab8_2_room_albums, PID: 8775
java.lang.IllegalStateException: Room cannot verify the data integrity. Looks like you've changed schema but forgot to update the version number. You can simply fix this by increasing the version number.
at androidx.room.RoomOpenHelper.checkIdentity(RoomOpenHelper.java:154)
at androidx.room.RoomOpenHelper.onOpen(RoomOpenHelper.java:135)
at androidx.sqlite.db.framework.FrameworkSQLiteOpenHelper$OpenHelper.onOpen(FrameworkSQLiteOpenHelper.java:142)
at android.database.sqlite.SQLiteOpenHelper.getDatabaseLocked(SQLiteOpenHelper.java:427)
at android.database.sqlite.SQLiteOpenHelper.getWritableDatabase(SQLiteOpenHelper.java:316)
at androidx.sqlite.db.framework.FrameworkSQLiteOpenHelper$OpenHelper.getWritableSupportDatabase(FrameworkSQLiteOpenHelper.java:92)
at androidx.sqlite.db.framework.FrameworkSQLiteOpenHelper.getWritableDatabase(FrameworkSQLiteOpenHelper.java:53)
at androidx.room.RoomDatabase.inTransaction(RoomDatabase.java:476)
at androidx.room.RoomDatabase.assertNotSuspendingTransaction(RoomDatabase.java:281)
at com.example.lab8_2_room_albums.dao.WordDao_Impl.insert(WordDao_Impl.java:57)
at com.example.lab8_2_room_albums.MainActivity.lambda$onCreate$0(MainActivity.java:59)
at com.example.lab8_2_room_albums.-$$Lambda$MainActivity$h9d6n2GFmqhE1uxj4ezb0-bRXOU.run(Unknown Source:2)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1167)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:641)
at java.lang.Thread.run(Thread.java:923)
I/Process: Sending signal. PID: 8775 SIG: 9
It refers to the line in the MainActivity where I run dao.insert(word) also where I insert a word. into database.
I did clean the project, rebuld the project too, but I still get the same error. Why? How to figure out what's wrong? And how can I fix this?
The database class is like below:
package com.example.lab8_2_room_albums.db;
import android.content.Context;
import androidx.annotation.NonNull;
import androidx.room.Database;
import androidx.room.Room;
import androidx.room.RoomDatabase;
import androidx.sqlite.db.SupportSQLiteDatabase;
import com.example.lab8_2_room_albums.dao.PhotoDAO;
import com.example.lab8_2_room_albums.dao.UserDAO;
import com.example.lab8_2_room_albums.dao.WordDao;
import com.example.lab8_2_room_albums.entities.Address;
import com.example.lab8_2_room_albums.entities.Album;
import com.example.lab8_2_room_albums.entities.AlbumPhotoCrossRef;
import com.example.lab8_2_room_albums.entities.Company;
import com.example.lab8_2_room_albums.entities.Geo;
import com.example.lab8_2_room_albums.entities.Photo;
import com.example.lab8_2_room_albums.entities.User;
import com.example.lab8_2_room_albums.entities.Word;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
#Database(entities = {Word.class, User.class, Address.class, Geo.class, Company.class, Album.class, Photo.class, AlbumPhotoCrossRef.class}, version = 1, exportSchema = true)
public abstract class UserRoomDatabase extends RoomDatabase {
//public abstract UserDAO userDAO();
// public abstract PhotoDAO photoDAO();
public abstract WordDao wordDao();
// volatile: har sammenheng med multithreading. Sikrer at alle trĂ¥der ser samme kopi av INSTANCE.
private static volatile UserRoomDatabase INSTANCE;
private static final int NUMBER_OF_THREADS = 4;
public static final ExecutorService databaseWriteExecutor =
Executors.newFixedThreadPool(NUMBER_OF_THREADS);
public static UserRoomDatabase getDatabase(final Context context) {
if (INSTANCE == null) {
synchronized (UserRoomDatabase.class) {
INSTANCE = Room.databaseBuilder(context.getApplicationContext(),
UserRoomDatabase.class, "mydaaatabase")
.addCallback(sRoomDatabaseCallback)
.build();
}
}
return INSTANCE;
}
private static RoomDatabase.Callback sRoomDatabaseCallback = new RoomDatabase.Callback() {
/**
* Called when the database is created for the first time.
* This is called after all the tables are created.
* #param db
*/
#Override
public void onCreate(#NonNull SupportSQLiteDatabase db) {
super.onCreate(db);
// If you want to keep data through app restarts,
// comment out the following block
databaseWriteExecutor.execute(() -> {
// Populate the database in the background.
WordDao wordDao = INSTANCE.wordDao();
//wordDao.deleteAll();
wordDao.insert(new Word("asd"));
wordDao.insert(new Word("adsds"));
wordDao.getAlphabetizedWords();
});
}
#Override
public void onOpen(#NonNull SupportSQLiteDatabase db) {
super.onOpen(db);
}
};
/*
static final Migration MIGRATION_1_2 = new Migration(1, 2) {
#Override
public void migrate(SupportSQLiteDatabase database) {
database.execSQL("ALTER TABLE user "
+ " ADD COLUMN birth_year INTEGER");
}
};*/
}
And here's the Main Activity:
package com.example.lab8_2_room_albums;
import androidx.appcompat.app.AppCompatActivity;
import androidx.lifecycle.ViewModelProvider;
import androidx.room.Room;
import android.os.Bundle;
import android.view.LayoutInflater;
import android.view.View;
import android.widget.Toast;
import com.example.lab8_2_room_albums.dao.PhotoDAO;
import com.example.lab8_2_room_albums.dao.UserDAO;
import com.example.lab8_2_room_albums.dao.WordDao;
import com.example.lab8_2_room_albums.databinding.ActivityMainBinding;
import com.example.lab8_2_room_albums.db.UserRoomDatabase;
import com.example.lab8_2_room_albums.entities.Address;
import com.example.lab8_2_room_albums.entities.Album;
import com.example.lab8_2_room_albums.entities.AlbumWithPhotos;
import com.example.lab8_2_room_albums.entities.Company;
import com.example.lab8_2_room_albums.entities.Geo;
import com.example.lab8_2_room_albums.entities.Photo;
import com.example.lab8_2_room_albums.entities.User;
import com.example.lab8_2_room_albums.entities.UserWithAlbums;
import com.example.lab8_2_room_albums.entities.UserWithCompanyWithAddressWithGeo;
import com.example.lab8_2_room_albums.entities.Word;
import com.example.lab8_2_room_albums.viewmodel.UserAlbumsViewModel;
public class MainActivity extends AppCompatActivity {
private UserAlbumsViewModel userAlbumsViewModel;
private ActivityMainBinding activityMainBinding;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
// ViewBinding:
LayoutInflater layoutInflater = (LayoutInflater) getSystemService(LAYOUT_INFLATER_SERVICE);
activityMainBinding = ActivityMainBinding.inflate(layoutInflater);
setContentView(activityMainBinding.getRoot());
// userAlbumsViewModel = new ViewModelProvider(this).get(UserAlbumsViewModel.class);
UserRoomDatabase db = UserRoomDatabase.getDatabase(this);
WordDao dao = db.wordDao();
UserRoomDatabase.databaseWriteExecutor.execute(() -> {
// Populate the database in the background.
// If you want to start with more words, just add them.
Word word = new Word("Helloooo");
dao.insert(word);
word = new Word("Woooorld");
dao.insert(word);
dao.getAlphabetizedWords();
});
}
}
I add some words into the words table in database. It works without any problem.
Word class:
package com.example.lab8_2_room_albums.entities;
import androidx.annotation.NonNull;
import androidx.room.ColumnInfo;
import androidx.room.Entity;
import androidx.room.PrimaryKey;
#Entity(tableName = "word_table")
public class Word {
#PrimaryKey
#NonNull
#ColumnInfo(name = "word")
private String mWord;
public Word(#NonNull String word) {this.mWord = word;}
public String getWord(){return this.mWord;}
}
WordDAO:
package com.example.lab8_2_room_albums.dao;
import androidx.lifecycle.LiveData;
import androidx.room.Dao;
import androidx.room.Insert;
import androidx.room.OnConflictStrategy;
import androidx.room.Query;
import com.example.lab8_2_room_albums.entities.Word;
import java.util.List;
#Dao
public interface WordDao {
// allowing the insert of the same word multiple times by passing a
// conflict resolution strategy
#Insert(onConflict = OnConflictStrategy.IGNORE)
void insert(Word word);
#Query("DELETE FROM word_table")
void deleteAll();
#Query("SELECT * FROM word_table ORDER BY word ASC")
LiveData<List<Word>> getAlphabetizedWords();
}
User class:
package com.example.lab8_2_room_albums.entities;
import androidx.annotation.NonNull;
import androidx.room.Entity;
import androidx.room.ForeignKey;
import androidx.room.PrimaryKey;
#Entity(foreignKeys = {
#ForeignKey(entity = Address.class, parentColumns = "addressId", childColumns = "fk_addressId", onDelete = ForeignKey.CASCADE),
#ForeignKey(entity = Company.class, parentColumns = "companyId", childColumns = "fk_companyId", onDelete = ForeignKey.CASCADE)
})
public class User {
#PrimaryKey(autoGenerate = true)
public long userId;
public String name;
public String username;
public String email;
public long fk_addressId;
public String phone;
public String website;
public long fk_companyId;
public User(#NonNull String name, #NonNull String username, #NonNull String email, long fk_addressId,
#NonNull String phone, #NonNull String website, long fk_companyId) {
this.name = name;
this.username = username;
this.email = email;
this.fk_addressId = fk_addressId;
this.phone = phone;
this.website = website;
this.fk_companyId = fk_companyId;
}
}
Address:
package com.example.lab8_2_room_albums.entities;
import androidx.annotation.NonNull;
import androidx.room.Entity;
import androidx.room.ForeignKey;
import androidx.room.PrimaryKey;
#Entity(foreignKeys = {
#ForeignKey(entity = Geo.class, parentColumns = "geoId", childColumns = "fk_geoId", onDelete = ForeignKey.CASCADE)
})
public class Address {
#PrimaryKey(autoGenerate = true)
public long addressId;
public String street;
public String suite;
public String city;
public String zipCode;
public long fk_geoId;
public Address(#NonNull String street, #NonNull String suite, #NonNull String city,
#NonNull String zipCode, long fk_geoId) {
this.street = street;
this.suite = suite;
this.city = city;
this.zipCode = zipCode;
this.fk_geoId = fk_geoId;
}
}
Geo:
package com.example.lab8_2_room_albums.entities;
import androidx.annotation.NonNull;
import androidx.room.Entity;
import androidx.room.PrimaryKey;
#Entity
public class Geo {
#PrimaryKey(autoGenerate = true)
public long geoId;
public double lat;
public double lng;
public Geo(#NonNull double lat, #NonNull double lng) {
this.lat = lat;
this.lng = lng;
}
}
Company:
package com.example.lab8_2_room_albums.entities;
import androidx.annotation.NonNull;
import androidx.room.Entity;
import androidx.room.PrimaryKey;
#Entity
public class Company {
#PrimaryKey(autoGenerate = true)
public long companyId;
public String name;
public String catchPhrase;
public String bs;
public Company(#NonNull String name, #NonNull String catchPhrase, #NonNull String bs) {
this.name = name;
this.catchPhrase = catchPhrase;
this.bs = bs;
}
}
Album:
package com.example.lab8_2_room_albums.entities;
import androidx.annotation.NonNull;
import androidx.room.Entity;
import androidx.room.ForeignKey;
import androidx.room.PrimaryKey;
#Entity(foreignKeys = {
#ForeignKey(entity = User.class, parentColumns="userId", childColumns = "fk_userId", onDelete = ForeignKey.CASCADE)
})
public class Album {
#PrimaryKey(autoGenerate = true)
public long albumId;
public long fk_userId;
public String title;
public Album(#NonNull String title, long fk_userId) {
this.title = title;
this.fk_userId = fk_userId;
}
}
Photo:
package com.example.lab8_2_room_albums.entities;
import androidx.annotation.NonNull;
import androidx.room.Entity;
import androidx.room.PrimaryKey;
#Entity
public class Photo {
#PrimaryKey(autoGenerate = true)
public long photoId;
public String title;
public String url;
public String thumbnailUrl;
public Photo(#NonNull String title, #NonNull String url, #NonNull String thumbnailUrl) {
this.title = title;
this.url = url;
this.thumbnailUrl = thumbnailUrl;
}
}
AlbumPhotoCrossRef:
package com.example.lab8_2_room_albums.entities;
import androidx.room.Entity;
import androidx.room.ForeignKey;
#Entity(primaryKeys = {"albumId", "photoId"},
foreignKeys = {
#ForeignKey(entity = Album.class, parentColumns="albumId", childColumns = "albumId", onDelete = ForeignKey.CASCADE),
#ForeignKey(entity = Photo.class, parentColumns="photoId", childColumns = "photoId", onDelete = ForeignKey.CASCADE)
}
)
public class AlbumPhotoCrossRef {
public long albumId;
public long photoId;
}

You've changed the schema (structure) without changing the version number. Room's integrity checking has detected the change and hence the exception.
As you have MIGRATION_1_2 you probably just need to change the version number from 1 to 2 as per :-
#Database(entities = {Word.class, User.class, Address.class, Geo.class, Company.class, Album.class, Photo.class, AlbumPhotoCrossRef.class}, version = 2, exportSchema = true
The message in the log basically says the above bar about having a migration available as per :-
java.lang.IllegalStateException: Room cannot verify the data integrity. Looks like you've changed schema but forgot to update the version number. You can simply fix this by increasing the version number.
However, looking at the migration it's altering the user table by adding a column. This doesn't appear to cover the changes you say you have made (added Word entity/table). You may wish to consider the alternative.
The alternative is to uninstall the App and rerun and the new schema will be applied (not any good if the App has been published as any data will be lost).
Room undertakes this checking by the use of a table called room_master_table this table stores an identity hash which is generated according to the schema that room builds/maintains from the code. Change the schema (code) and the hash changes. If the hash mismatches and the version number is not changed then the exception (or similar in the case of down grade)

Related

how to Initialization sequence generator _id in spring data mongo

i'm using spring boot + spring data mongodb and set Auto Generator Service ...
import lombok.Getter;
import lombok.Setter;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
#Getter
#Setter
#Document(collection = "auto_sequence")
public class AutoIncrementSequence {
#Id
private String id;
private Long seq;
}
and
package com.juhyun.shorturl.entity.sequence;
import lombok.RequiredArgsConstructor;
import org.springframework.data.mongodb.core.MongoOperations;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Service;
import java.util.Objects;
import static org.springframework.data.mongodb.core.FindAndModifyOptions.options;
import static org.springframework.data.mongodb.core.query.Criteria.where;
#RequiredArgsConstructor
#Service
public class SequenceGeneratorService {
private final MongoOperations mongoOperations;
public Long generateSequence(String key) {
AutoIncrementSequence counter = mongoOperations.findAndModify(Query.query(where("_id").is(key)),
new Update().inc("seq", 1), options().returnNew(true).upsert(true), AutoIncrementSequence.class);
//return BigInteger.valueOf(!Objects.isNull(counter) ? counter.getSeq() : 1);
return !Objects.isNull(counter) ? counter.getSeq() : 1;
}
}
Entity
public class ShortUrl {
#Transient
public static final String SEQUENCE_NAME = "shorturl_sequence";
#Id
private Long _id;
If the return value of the generateSequence method of class SequenceGeneratorService is changed to 1L, it will be solved, but I wonder if there is any other way.
Thank you :)
Change return value only 1L in SequenceGeneratorService generateSequence method
Change sequence number in collection (I was using mongodb compass)

UUID as _id in mongodb

I've found multiple sites saying using UUID is possible, but not how. I've tried to simply have a "_id" property that is a UUID, but when I use postman to get the object the "_id" field says null.
Here is a screenshot.
I've made sure mongodb knows UUID by specifying it in the settings:
`
import com.mongodb.ConnectionString;
import com.mongodb.MongoClientSettings;
import com.mongodb.client.*;
import static com.mongodb.client.model.Filters.eq;
import com.mongodb.client.model.FindOneAndReplaceOptions;
import com.mongodb.client.model.ReturnDocument;
import de.tudo.ls14.aqua.smarthome.model.Device;
import de.tudo.ls14.aqua.smarthome.model.Household;
import de.tudo.ls14.aqua.smarthome.model.User;
import org.bson.Document;
import org.bson.UuidRepresentation;
import org.bson.codecs.configuration.CodecRegistry;
import org.bson.codecs.pojo.PojoCodecProvider;
import org.springframework.stereotype.Repository;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
import static org.bson.codecs.configuration.CodecRegistries.fromProviders;
import static org.bson.codecs.configuration.CodecRegistries.fromRegistries;
#Repository("Mongodao")
public class MongoDao {
final MongoClient mongoClient;
final MongoDatabase mongoDatabase;
final MongoCollection<User> userCollection;
final MongoCollection<Household> householdCollection;
final MongoCollection<Device> deviceCollection;
public MongoDao() {
String password = System.getProperty("password");//Passwort aus den VM options
Logger.getLogger("org.mongodb.driver").setLevel(Level.ALL);
ConnectionString connectionString = new ConnectionString("someConnectionString");
CodecRegistry pojoCodecRegistry = fromProviders(PojoCodecProvider.builder().automatic(true).build());
CodecRegistry codecRegistry = fromRegistries(MongoClientSettings.getDefaultCodecRegistry(), pojoCodecRegistry);
MongoClientSettings clientSettings = MongoClientSettings.builder()
.uuidRepresentation(UuidRepresentation.STANDARD)
.applyConnectionString(connectionString)
.codecRegistry(codecRegistry)
.build();
mongoClient = MongoClients.create(clientSettings);
mongoDatabase = mongoClient.getDatabase("ProjektDB");
userCollection = mongoDatabase.getCollection("userCollection", User.class);
householdCollection = mongoDatabase.getCollection("householdCollection", Household.class);
deviceCollection = mongoDatabase.getCollection("deviceCollection", Device.class);
}
public User getUserById(UUID id) {
return userCollection.find(eq("id", id)).first();
}
public Household getHouseholdById(UUID id) {
return householdCollection.find(eq("id", id)).first();
}
public Device getDeviceById(UUID id) {
return deviceCollection.find(eq("id", id)).first();
}
public int addHousehold(Household household) {
householdCollection.insertOne(household);
return 1;
}
public int addUser(User user) {
userCollection.insertOne(user);
System.out.println(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> added:"+user.toString());
return 1;
}
public int addDevice(Device device) {
deviceCollection.insertOne(device);
return 1;
}
public User updateUserById(User user) {
Document filterByUserId = new Document("_id", user.get_id());
FindOneAndReplaceOptions returnDocAfterReplace = new FindOneAndReplaceOptions().returnDocument(ReturnDocument.AFTER);
return userCollection.findOneAndReplace(filterByUserId, user, returnDocAfterReplace);
}
//nur zum testen
public List<User> getAllUsers() {
MongoCursor<User> cursor = userCollection.find().iterator();
List<User> userList = new ArrayList<>();
try{
while(cursor.hasNext()){
userList.add(cursor.next());
}
} finally {
cursor.close();
}
return userList;
}
}
`
Funny enough when i check with ATLAS it even says the _id field is an objectid as seen here.
I've printed my object out and it is clear that the _id field is UUID as seen here.
For the sake of it here is also shown that my pojo certainly uses UUID for the _id field. I'm out of ideas.
Do I have to specify things differently?
I had to add a #BsonProperty("_id") annotation to the UUID property in the POJO because it was just private UUID id = IdUtils.newId(); Not _id.
To query using the java driver:
import org.bson.Document;
public User findUserById(final UUID id){
return collection.find(new Document("_id", id)).first();
}

Apache beam 2.1.0: Unable to upload to Datastore after following example

I'm having trouble uploading entities to the Cloud Datastore via the Apache Beam Java SDK (2.1.0). The following is my code:
import com.google.cloud.datastore.DatastoreOptions
import com.google.cloud.datastore.Entity
import com.opencsv.CSVParser
import org.apache.beam.runners.dataflow.DataflowRunner
import
org.apache.beam.runners.dataflow.options.DataflowPipelineOptions
import org.apache.beam.sdk.Pipeline
import org.apache.beam.sdk.io.TextIO
import org.apache.beam.sdk.io.gcp.datastore.DatastoreIO
import org.apache.beam.sdk.options.PipelineOptionsFactory
import org.apache.beam.sdk.transforms.DoFn
import org.apache.beam.sdk.transforms.MapElements
import org.apache.beam.sdk.transforms.ParDo
import org.apache.beam.sdk.transforms.SimpleFunction
import java.io.Serializable
object PipelineClass {
class FoodGroup(var id: String? = null,
var group: String? = null) : Serializable
class CreateGroupsFn : SimpleFunction<String, FoodGroup>() {
override fun apply(line: String?): FoodGroup {
val group = FoodGroup()
val parser = CSVParser()
val parts = parser.parseLine(line)
group.id = parts[0].trim()
group.group = parts[1].trim()
return group
}
}
class CreateEntitiesFn : DoFn<FoodGroup, Entity>() {
#ProcessElement
fun processElement(c: ProcessContext) {
val datastore = DatastoreOptions.getDefaultInstance().service
val keyFactory = datastore.newKeyFactory()
.setKind("FoodGroup")
.setNamespace("nutrients")
val key = datastore.allocateId(keyFactory.newKey())
val entity = Entity.newBuilder(key)
.set("id", c.element().id)
.set("group", c.element().group)
.build()
c.output(entity)
}
}
#JvmStatic fun main(args: Array<String>) {
val options =
PipelineOptionsFactory.`as`(DataflowPipelineOptions::class.java)
options.runner = DataflowRunner::class.java
options.project = "simplesample"
options.jobName = "fgUpload"
val pipeline = Pipeline.create(options)
pipeline.apply(TextIO.read().from("gs://bucket/foodgroup.csv"))
.apply(MapElements.via(CreateGroupsFn()))
.apply(ParDo.of(CreateEntitiesFn()))
//error occurs below...
.apply(DatastoreIO.v1().write()
.withProjectId(options.project))
pipeline.run()
}
}
The following is the error I get:
PipelineClass.kt: (75, 24): Type mismatch: inferred type is
DatastoreV1.Write! but PTransform<in PCollection<Entity!>!, PDone!>!
was expected
I've tried SimpleFunction, DoFn, and PTransform (composite and non-composite) to do the transform from String to Entity with no success.
What am I doing wrong?
EDIT: I've finally managed to get my entities in the Datastore. I decided to use Dataflow 1.9.1 and ditched Beam (2.1.0) after seeing this example. Below is my code:
import com.google.cloud.dataflow.sdk.Pipeline;
import com.google.cloud.dataflow.sdk.io.TextIO;
import com.google.cloud.dataflow.sdk.io.datastore.DatastoreIO;
import com.google.cloud.dataflow.sdk.options.Default;
import com.google.cloud.dataflow.sdk.options.Description;
import com.google.cloud.dataflow.sdk.options.PipelineOptions;
import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory;
import com.google.cloud.dataflow.sdk.transforms.DoFn;
import com.google.cloud.dataflow.sdk.transforms.ParDo;
import com.google.datastore.v1.Entity;
import com.google.datastore.v1.Key;
import com.opencsv.CSVParser;
import javax.annotation.Nullable;
import java.util.UUID;
import static com.google.datastore.v1.client.DatastoreHelper.makeKey;
import static
com.google.datastore.v1.client.DatastoreHelper.makeValue;
public class PipelineClass {
static class CreateEntitiesFn extends DoFn<String, Entity> {
private final String namespace;
private final String kind;
private final Key ancestorKey;
CreateEntitiesFn(String namespace, String kind) {
this.namespace = namespace;
this.kind = kind;
ancestorKey = makeAncestorKey(namespace, kind);
}
Entity makeEntity(String id, String group) {
Entity.Builder entityBuilder = Entity.newBuilder();
Key.Builder keyBuilder = makeKey(ancestorKey, kind,
UUID.randomUUID().toString());
if (namespace != null) {
keyBuilder.getPartitionIdBuilder().setNamespaceId(namespace);
}
entityBuilder.setKey(keyBuilder.build());
entityBuilder.getMutableProperties().put("id",
makeValue(id).build());
entityBuilder.getMutableProperties().put("group",
makeValue(group).build());
return entityBuilder.build();
}
#Override
public void processElement(ProcessContext c) throws Exception {
CSVParser parser = new CSVParser();
String[] parts = parser.parseLine(c.element());
String id = parts[0];
String group = parts[1];
c.output(makeEntity(id, group));
}
}
static Key makeAncestorKey(#Nullable String namespace, String kind) {
Key.Builder keyBuilder = makeKey(kind, "root");
if (namespace != null) {
keyBuilder.getPartitionIdBuilder().setNamespaceId(namespace);
}
return keyBuilder.build();
}
public interface Options extends PipelineOptions {
#Description("Path of the file to read from and store to Cloud
Datastore")
#Default.String("gs://bucket/foodgroup.csv")
String getInput();
void setInput(String value);
#Description("Dataset ID to read from Cloud Datastore")
#Default.String("simplesample")
String getProject();
void setProject(String value);
#Description("Cloud Datastore Entity Kind")
#Default.String("FoodGroup")
String getKind();
void setKind(String value);
#Description("Dataset namespace")
#Default.String("nutrients")
String getNamespace();
void setNamespace(#Nullable String value);
#Description("Number of output shards")
#Default.Integer(0)
int getNumShards();
void setNumShards(int value);
}
public static void main(String args[]) {
PipelineOptionsFactory.register(Options.class);
Options options =
PipelineOptionsFactory.fromArgs(args).as(Options.class);
Pipeline p = Pipeline.create(options);
p.apply(TextIO.Read.named("ReadLines").from(options.getInput()))
.apply(ParDo.named("CreateEntities").of(new
CreateEntitiesFn(options.getNamespace(), options.getKind())))
.apply(DatastoreIO.v1().write().withProjectId(options.getProject()));
p.run();
}
}

JDO + RequestFactory - entity versioning

I was trying to set-up a really trivial RequestFactory example, but I failed. I persist entities in to my datastore just find, however when trying to pull them out again, i get a
com.google.web.bindery.requestfactory.server.UnexpectedException: The persisted entity with id aglub19hcHBfaWRyCgsSBFVzZXIYBAw has a null version
So first of all this is my JPO annotated entity class. At the end you find to static function for RequestFactory to call, and a non-static member function which will become an InstanceRequest.
package com.test.server;
import java.util.List;
import javax.jdo.PersistenceManager;
import javax.jdo.annotations.Column;
import javax.jdo.annotations.Extension;
import javax.jdo.annotations.IdGeneratorStrategy;
import javax.jdo.annotations.IdentityType;
import javax.jdo.annotations.PersistenceCapable;
import javax.jdo.annotations.Persistent;
import javax.jdo.annotations.PrimaryKey;
import javax.jdo.annotations.Version;
import javax.jdo.annotations.VersionStrategy;
#PersistenceCapable(identityType = IdentityType.APPLICATION)
#Version(strategy = VersionStrategy.VERSION_NUMBER, column = "VERSION", extensions = { #Extension(vendorName = "datanucleus", key = "field-name", value = "version") })
public class User {
public User() {
}
public User(String name) {
this.name = name;
}
#PrimaryKey
#Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
#Extension(vendorName = "datanucleus", key = "gae.encoded-pk", value = "true")
private String id;
#Persistent
#Column(name = "version")
private Integer version;
#Persistent
private String name;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public Integer getVersion() {
return version;
}
public void setVersion(Integer version) {
this.version = version;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public static final PersistenceManager persistenceManager() {
return PMF.get().getPersistenceManager();
}
#SuppressWarnings("unchecked")
public static List<User> findAllUsers() {
PersistenceManager pm = persistenceManager();
try {
String query = "SELECT FROM " + User.class.getName();
List<User> objects = (List<User>) pm.newQuery(query).execute();
objects.size(); // This is the workaround to retrieve all objects
return objects;
} finally {
pm.close();
}
}
public static User findUser(String id) {
PersistenceManager pm = persistenceManager();
try {
User u = pm.getObjectById(User.class, id);
return u;
} finally {
pm.close();
}
}
public void persist() {
PersistenceManager pm = persistenceManager();
try {
pm.makePersistent(this);
} finally {
pm.close();
}
}
}
The RequestFactory interface itself is really simple
package com.test.shared;
import com.google.web.bindery.requestfactory.shared.RequestFactory;
public interface UserOrderRequestFactory extends RequestFactory {
UserRequest userRequest();
}
so is the corresponding RequestContext
package com.test.shared;
import java.util.List;
import com.google.web.bindery.requestfactory.shared.InstanceRequest;
import com.google.web.bindery.requestfactory.shared.RequestContext;
import com.google.web.bindery.requestfactory.shared.Service;
import com.google.web.bindery.requestfactory.shared.Request;
import com.test.server.User;
#Service(User.class)
public interface UserRequest extends RequestContext {
Request<List<UserProxy>> findAllUsers();
InstanceRequest<UserProxy, Void> persist();
}
Here is the proxy of user for the client side
package com.test.shared;
import com.google.web.bindery.requestfactory.shared.EntityProxy;
import com.google.web.bindery.requestfactory.shared.EntityProxyId;
import com.google.web.bindery.requestfactory.shared.ProxyFor;
#ProxyFor(com.test.server.User.class)
public interface UserProxy extends EntityProxy {
EntityProxyId<UserProxy> stableId();
String getName();
void setName(String name);
}
and finally my onModuleLoad() which first persists a user and then gets a list of all users.
public void onModuleLoad() {
final EventBus eventBus = new SimpleEventBus();
requestFactory = GWT.create(UserOrderRequestFactory.class);
requestFactory.initialize(eventBus);
UserRequest userRequest = requestFactory.userRequest();
UserProxy user = userRequest.create(UserProxy.class);
user.setName("Luigi");
userRequest.persist().using(user).fire( new Receiver<Void>()
{
#Override
public void onSuccess(Void arg0)
{
GWT.log("User persisted.");
}
});
userRequest = requestFactory.userRequest();
Request<List<UserProxy>> findAllUsersRequest = userRequest.findAllUsers();
findAllUsersRequest.fire( new Receiver<List<UserProxy>>() {
#Override
public void onSuccess(List<UserProxy> list) {
for(UserProxy u: list) {
GWT.log(u.getName());
}
}
});
Any input is welcome. I would be happy to receive any advice on this.
Thank you in advance.
While JPA seems to do this automatically it seems to be my job to advance the version counter in JDO. I added the following code to my persist routine in User.java
// JPA #Version does this automatically, but JDO #Version is not working like that. Not sure why.
if (version == null) {
version = 0l;
}
version++;
I am not sure if it matters but the #Version column = "VERSION" does not match the #Column(name = "version").
However GAE does not really have 'columns' as such and you can just ignore them by removing the column = "" and the #Column.
See http://gae-java-persistence.blogspot.com.au/2009/10/optimistic-locking-with-version.html

Inconsistent cascading persistence-by-reachability behavior with JDO, App Engine, Data Nucleus, JUnit

I'm experimenting with App Engine, using JDO and DataNucleus for persistence. I have a simple domain that includes several unidirectional relationships. The question comes with nesting those relationships:
Civilization -(1-1)-> Clan
Civilization -(1-1)-> Land
Civilization -(1-1)-> Military -(1-N)-> Armies (this is inconsistent)
Civilization -(1-N)-> Settlement
According to the DataNucleus Documentation, persistence-by-reachability semantics should persist everything by cascading on a persist of a Civilization. I have a JUnit test to check the basic storage and retrieval of these objects, but its behavior is inconsistent. With no changes to the code, repeated runs of the test give nondeterministic results. Specifically, the armies only persist about 50% of the time. They are the only test that fails.
I could more easily understand a scenario where Armies never persist, but the irregular behavior has me at a loss. Everything else persists correctly and consistently. I've tried wrapping the factory method in a transaction and I've tried bidirectional relationships, and neither have changed the 50/50 pass/fail split in JUnit.
I am using Annotation-based configuration for DataNucleus, as described in the App Engine documentation (link not included because of anti-spam measures). I apologize for the large amount of code attached; I just don't know where I'm going wrong.
CivilizationCreateTest.java:
package com.moffett.grunzke.server;
import static org.junit.Assert.*;
import java.util.ArrayList;
import java.util.List;
import javax.jdo.PersistenceManager;
import javax.jdo.Query;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.google.appengine.api.users.User;
import com.google.appengine.api.users.UserService;
import com.google.appengine.api.users.UserServiceFactory;
import com.google.appengine.tools.development.testing.LocalServiceTestHelper;
import com.moffett.grunzke.generic.GenericHelperFactory;
import com.moffett.grunzke.server.civilization.Army;
import com.moffett.grunzke.server.civilization.Civilization;
import com.moffett.grunzke.server.civilization.Clan;
import com.moffett.grunzke.server.civilization.Land;
import com.moffett.grunzke.server.civilization.Military;
import com.moffett.grunzke.server.civilization.Settlement;
#SuppressWarnings("unchecked")
public class CivilizationCreationTest
{
private final LocalServiceTestHelper helper = new LocalServiceTestHelper(
new LocalDatastoreServiceTestConfig(),
new LocalUserServiceTestConfig())
.setEnvIsLoggedIn(true)
.setEnvEmail("generic.user#gmail.com")
.setEnvAuthDomain("google.com");
#Before
public void setUp()
{
helper.setUp();
}
#After
public void tearDown()
{
helper.tearDown();
}
#Test
public void testCivilizationCreation()
{
String clanName = "Test Clan";
String rulerName = "Test Ruler";
UserService userService = UserServiceFactory.getUserService();
User user = userService.getCurrentUser();
if (user == null)
{
fail("No user");
}
PersistenceManager pm = PMF.get().getPersistenceManager();
CivilizationFactory.newInstance(user, clanName, rulerName);
// We check to make sure that 1, and only 1 Civilization has been made.
Query q1 = pm.newQuery("SELECT FROM " + Civilization.class.getName());
List<Civilization> allCivilizations = (List<Civilization>) q1.execute();
assertTrue(allCivilizations.size() == 1);
// Now we move on to checking the other aspects.
Civilization persistentCiv = allCivilizations.get(0);
Clan persistentClan = persistentCiv.getClan();
Land persistentLand = persistentCiv.getLand();
Military persistentMilitary = persistentCiv.getMilitary();
ArrayList<Settlement> persistentSettlements = persistentCiv.getSettlements();
// Make sure Civ has pointers to all the necessary elements.
assertTrue(persistentClan != null);
assertTrue(persistentLand != null);
assertTrue(persistentMilitary != null);
assertTrue(persistentMilitary.getArmies() != null);
assertTrue(persistentSettlements != null);
// Lastly we want to make sure that there is only one entry in each of Clan,
// Land, Military, Army, Settlement.
Query q2 = pm.newQuery("SELECT FROM " + Clan.class.getName());
List<Clan> allClans = (List<Clan>) q2.execute();
assertTrue(allClans.size() == 1);
Query q3 = pm.newQuery("SELECT FROM " + Land.class.getName());
List<Land> allLand = (List<Land>) q3.execute();
assertTrue(allLand.size() == 1);
Query q4 = pm.newQuery("SELECT FROM " + Military.class.getName());
List<Military> allMilitary = (List<Military>) q4.execute();
assertTrue(allMilitary.size() == 1);
Query q5 = pm.newQuery("SELECT FROM " + Army.class.getName());
List<Army> allArmy = (List<Army>) q5.execute();
// *** THIS FAILS 50% OF THE TIME ***
assertTrue(allArmy.size() == 1);
Query q6 = pm.newQuery("SELECT FROM " + Settlement.class.getName());
List<Settlement> allSettlement = (List<Settlement>) q6.execute();
assertTrue(allSettlement.size() == 1);
}
}
CivilizationFactory.java:
package com.moffett.grunzke.server;
import java.util.ArrayList;
import com.google.appengine.api.users.User;
import com.moffett.grunzke.server.civilization.Army;
import com.moffett.grunzke.server.civilization.Civilization;
import com.moffett.grunzke.server.civilization.Clan;
import com.moffett.grunzke.server.civilization.Land;
import com.moffett.grunzke.server.civilization.Military;
import com.moffett.grunzke.server.civilization.Settlement;
public class CivilizationFactory
{
public static Civilization newInstance(User user, String clanName, String rulerName)
{
// First we make a new clan.
Clan clan = new Clan();
clan.setUser(user);
clan.setClanName(clanName);
clan.setRulerName(rulerName);
// Don't need land.save() because of persistence-by-reachability
// Now we need to make a new Land.
Land land = new Land();
land.setArableLand(100);
land.setPasturableLand(0);
land.setLandUsedBySettlements(0);
// Don't need land.save() because of persistence-by-reachability
// Now we need to make a new Military
Military military = new Military();
Army army = new Army();
army.setMeleeUnits(10);
army.setRangedUnits(10);
army.setMountedUnits(10);
military.addArmy(army);
// Don't need military.save() because of persistence-by-reachability
// Now we need to make a new Settlement
Settlement settlement = new Settlement();
// Don't need settlement.save() because of persistence-by-reachability
ArrayList<Settlement> settlements = new ArrayList<Settlement>();
settlements.add(settlement);
// Lastly join everything together in the civ
Civilization civ = new Civilization();
civ.setClan(clan);
civ.setLand(land);
civ.setMilitary(military);
civ.setSettlements(settlements);
civ.save();
// civ.save should casacde to cover all of the elements above
return civ;
}
}
Civilization.java:
package com.moffett.grunzke.server.civilization;
import java.util.ArrayList;
import javax.jdo.PersistenceManager;
import javax.jdo.annotations.IdGeneratorStrategy;
import javax.jdo.annotations.PersistenceCapable;
import javax.jdo.annotations.Persistent;
import javax.jdo.annotations.PrimaryKey;
import com.google.appengine.api.datastore.Key;
import com.moffett.grunzke.server.PMF;
#PersistenceCapable
public class Civilization
{
#PrimaryKey
#Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
private Key key;
#Persistent
private Clan clan;
#Persistent
private Land land;
#Persistent
private Military military;
#Persistent
private ArrayList<Settlement> settlements = new ArrayList<Settlement>();
public void save()
{
PersistenceManager pm = PMF.get().getPersistenceManager();
try
{
pm.makePersistent(this);
}
finally
{
pm.close();
}
}
public ArrayList<Settlement> getSettlements()
{
return settlements;
}
public void setSettlements(ArrayList<Settlement> settlements)
{
this.settlements = settlements;
}
public Key getKey()
{
return key;
}
public void setKey(Key key)
{
this.key = key;
}
public Clan getClan()
{
return clan;
}
public void setClan(Clan clan)
{
this.clan = clan;
}
public Land getLand()
{
return land;
}
public void setLand(Land land)
{
this.land = land;
}
public void setMilitary(Military military)
{
this.military = military;
}
public Military getMilitary()
{
return military;
}
}
Military.java
package com.moffett.grunzke.server.civilization;
import java.util.ArrayList;
import javax.jdo.annotations.IdGeneratorStrategy;
import javax.jdo.annotations.PersistenceCapable;
import javax.jdo.annotations.Persistent;
import javax.jdo.annotations.PrimaryKey;
import com.google.appengine.api.datastore.Key;
#PersistenceCapable
public class Military
{
#PrimaryKey
#Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
private Key key;
#Persistent
private ArrayList<Army> armies = new ArrayList<Army>();
public Key getKey()
{
return key;
}
public void setKey(Key key)
{
this.key = key;
}
public ArrayList<Army> getArmies()
{
return armies;
}
public void setArmies(ArrayList<Army> armies)
{
this.armies = armies;
}
public void addArmy(Army army)
{
this.armies.add(army);
}
}
Army.java
package com.moffett.grunzke.server.civilization;
import javax.jdo.annotations.IdGeneratorStrategy;
import javax.jdo.annotations.PersistenceCapable;
import javax.jdo.annotations.Persistent;
import javax.jdo.annotations.PrimaryKey;
import com.google.appengine.api.datastore.Key;
#PersistenceCapable
public class Army
{
#PrimaryKey
#Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
private Key key;
#Persistent
private int meleeUnits;
#Persistent
private int rangedUnits;
#Persistent
private int mountedUnits;
public Key getKey()
{
return key;
}
public void setKey(Key key)
{
this.key = key;
}
public int getMeleeUnits()
{
return meleeUnits;
}
public void setMeleeUnits(int meleeUnits)
{
this.meleeUnits = meleeUnits;
}
public int getRangedUnits()
{
return rangedUnits;
}
public void setRangedUnits(int rangeUnits)
{
this.rangedUnits = rangeUnits;
}
public int getMountedUnits()
{
return mountedUnits;
}
public void setMountedUnits(int mountedUnits)
{
this.mountedUnits = mountedUnits;
}
}
My guess is the problem is with the way you are setting the setter methods that take a List are implemented. Remember that JDO will replace the ArrayList fields with persistance-aware versions, so you don't want to change the fields. Try this:
#PersistenceCapable
public class Military {
#PrimaryKey
#Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
private Key key;
#Persistent
private final List <Army> armies = new ArrayList<Army>();
public void setArmies(List<Army> armies) {
this.armies.clear();
this.armies.addAll(armies);
}
This is a good idea for other reasons as well. You don't want someone doing this:
military.setArmies(armies);
armies.clear();
...or this:
military.getArmies().clear();
Personally, I would have the methods that change your entities expose only the operations you want:
public void addArmy(Army army) {
armies.add(army);
}
public List<Army> getArmies() {
return Collections.unmodifiableList(armies);
}

Resources