String Length changes if it is pulled from file - file

Question

I'm coding in Game Maker Studio (Toby Fox used it for undertale) and I'm starting to try and work with files. Effectivly, I'm trying to build a custom "level" editor because the built in one for GMS lacks some features I want. So far I have set most of it up, but I still have to make it regurgitate the saved "levels" from the files. It's almost completly working, but there's one problem. Here's some info you should know ahead of time:
GMS uses really nonstandard syntax. I apologize ahead of time for that.
GMS is weird so arrays don't work with JSON formatting
as a result of No. 2, I have coded my own JSON formatting, which I reffer to as GSON (Gamemaker Studio Object Notation)
I am using GSON not only to store levels (and their respective components) but I also want to build a copy paste functionality, which I would use GSON for (copy the component and it just gives you a GSON string which is then interpreted back when you paste it, so that I can copy and paste not only the component type (i.e. a solid vs the player), but the variable values as well).
So, into the meat of it...
global.lineBreak = "\n"
global.sectionSign = "§"
{ // gsonStringifySelf
function gsonStringifySelf(varnamearray, exception = undefined){
/*
"exception" should be set to the variable you will use to store the returned string. if you decide to instead just upload it
directly to the file without a middleman variable, then leave the field empty. e.g.
gsonstring = gsonStringifySelf(varnamearray, "gsonstring")
varnamearray should be an array with the names of all the variables you want to save. I could hardwire it to use
variable_instance_get_names(self)
but this doesn't get instance_variables, so instead its best to have an input, especially so you can leave out
instance variables you don't care about. If you want to put in all the instance variables,you have to mannually add their names to the array, which could be done via the following:
var names = variable_instance_get_names(self) // (var means it is a strictly local variable, so it can't be accessed by any other objects)
var i = array_length(names) // array indexes start at 0 but array_length starts at 1, so I is refrencing the unset index that is closest to index 0
names[i++] = "id" // i++ returns i **then** increments it, causing it to refrence the correct array index, as mentioned above.
names[i++] = "visible"
names[i++] = "solid"
names[i++] = "persistent"
names[i++] = "depth"
names[i++] = "layer"
names[i++] = "alarm"
names[i++] = "toString"
names[i++] = "direction"
names[i++] = "friction"
names[i++] = "gravity"
names[i++] = "gravity_direction"
names[i++] = "hspeed"
names[i++] = "vspeed"
names[i++] = "speed"
names[i++] = "xstart"
names[i++] = "ystart"
names[i++] = "x"
names[i++] = "y"
names[i++] = "xprevious"
names[i++] = "yprevious"
names[i++] = "image_xscale"
names[i++] = "image_yscale"
strvars = gsonStringifySelf(names, strvars)
i++ returns the value of i, then increments it.
The gson formatting works as follows:
first it tells you what kind of object it is:
<objectname>{\n
(\n is the code for "new line")
then it adds the variables, which are formatted as follows:
<type>:<variablename>:<value>\n
then it puts "}\n" on the very end,
all together it looks like this:
<objectname>{\n
<type>:<variablename>:<value>\n
}\n
the types are as follows:
A = Array (list of entries)
B = Boolian (true/false)
R = Real (any and all numbers, yes this is unstandard)
S = String (Letters and Characters)
U = Undefined (Built in Variable "Undefined")
so these two are effectivly equivalent:
number = 20
"R:number:20"
Arrays are a special case:
A:<arrayname>:{,<type>:<value1>,<type>:<value2>...}\n
In this way multi dimentional arrays are instead shown as nested one dimensional arrays.
Multi dimensional arrays and nested arrays respectively look like this:
2 dimensional array that is 2x1 in size
A:<arrayname>:{A:{<type>:<value>},A:{<type>:<value>}}\n
2 length 1 dimensional array, with a 2 length array nested inside
A:<arrayname>:{A:{<type>:<value>,<type>:<value>}, <type>:<value>}\n
I mention this because in a case like this:
a = [
10,
5,
false
]
b = [
a,
"nope"
]
there will be no connection between b[0] and a. Instead, b[0] will be only store the values, and as such changing b[0][1] will not affect the value of a[1]
note that in scenarios like this:
array[0] = 1
array[2] = "hello"
array[1] will also be part of the string, so it will look like this:
A:array:{R:1;R:0;S:hello}\n
This is because of a quirk with GMS. If you set a location in an array "past" an undefined location, then the undefined location is set to 0 when it is accessed.
*/
static lineBreak = "\n"
static sectionSign = "§"
var str = sectionSign + object_get_name(object_index) + "{" + lineBreak // Make the first line = "§<objectname>{\n"
var str = global.sectionSign + object_get_name(object_index) + "{" + global.lineBreak // Make the first line = "§<objectname>{\n"
var names = varnamearray
for(var i = 0; i < array_length(names); i++){
if(names[i] == string(exception)){
continue
}
var tempstr = ""
var r = variable_instance_get(self, names[i])
if(is_array(r)){
tempstr = "A:" + names[i] + ":"
tempstr += string(gsonStringifyArray(r)) + global.lineBreak
} else {
tempstr = gsonValueTypeNotate(r, names[i]) + global.lineBreak
}
str += tempstr
}
str += "}" + global.lineBreak
return str
}
function gsonValueTypeNotate(val, name){
if(name == undefined){
var a = ""
name = ""
} else {
var a = ":"
}
if(is_bool(val)){
var str = "B:" + name + a + string(val)
} else if(is_numeric(val)){
var str = "R:" + name + a + string(val)
} else if(is_string(val)){
var str = "S:" + name + a + val
} else if(is_undefined(val)){
var str = "U:" + name + a + string(val)
}
return str
}
function gsonStringifyArray(array){
var str = "{" + global.sectionSign + ","
var len = array_length(array)
var len1 = len - 1
for(var i = 0; i < len1; i++){
if(is_array(array[i])){
str += "A:" + gsonStringifyArray(array[i]) + global.sectionSign + ","
continue
} else {
str += gsonValueTypeNotate(array[i], undefined) + global.sectionSign + ","
}
}
if(is_array(array[i])){
str += gsonStringifyArray(array[i])
} else {
str += gsonValueTypeNotate(array[i], undefined)
}
str += global.sectionSign + ",}"
return str
}
}
{ // gsonParse
function gsonParseObject(str){
var array
array = stringLineify(str)
var a1 = array[0]
var a2 = string_length(a1)
var a = string_copy(array[0], 2, string_length(array[0]) - 3)
var b = instance_create_depth(0, 0, 0, asset_get_index(a))
var c
for(var i = 1; i < array_length(array) - 1; i++){
if(string_copy(array[i], 1, 1) = "A"){
for(var j = 3; string_copy(array[i], j, 1) != ":"; j++){
}
c[0] = string_copy(array[i], 3, j - 3)
c[1] = parseArrayValue(array[i])
variable_instance_set(b, c[0], c[1])
} else {
c = getVarNameValue(array[i])
if(string_copy(array[i], 1, 1) = "B"){
variable_instance_set(b, c[0], bool(c[1]))
} else if(string_copy(array[i], 1, 1) = "R"){
variable_instance_set(b, c[0], real(c[1]))
} else if(string_copy(array[i], 1, 1) = "S"){
variable_instance_set(b, c[0], c[1])
} else if(string_copy(array[i], 1, 1) = "U"){
variable_instance_set(b, c[0], undefined)
}
}
}
}
function getVarNameValue(str){
for(var i = 3; string_copy(str, i, 1) != ":"; i++){
}
var a
a[0] = string_copy(str, 3, i - 3)
var ind = ++i
for(; string_copy(str, i, 1) != global.lineBreak; i++){
}
a[1] = string_copy(str, ind, i - ind)
return a
}
function parseArrayValue(str){
var array
array[0] = ""
var l
var b = 0
var c = 1
for(var i = 2; i <= string_length(str); i++){
var l = string_copy(str, i, 1)
if(l == global.sectionSign){
i += 2
var l = string_copy(str, i, 1)
if(l == "A"){
c = i + 2
for(var j = c; string_copy(str, j, 1) != "}"; ++j){
}
show_debug_message(string_copy(str, c, ++j))
array[b++] = parseArrayValue(string_copy(str, c, j - c))
i = j - 1
} else if(l == "}"){
break
} else {
for(var j = i; string_copy(str, j, 1) != global.sectionSign and j <= string_length(str); ++j){
var l = string_copy(str, j, 1)
}
array[b++] = getArrValue(string_copy(str, i, j - i))
i = j - 1
}
}
}
return array
}
function getArrValue(str){
var v = string_copy(str, 3, string_length(str))
if(string_copy(str, 1, 1) = "B"){
return bool(v)
} else if(string_copy(str, 1, 1) = "R"){
return real(v)
} else if(string_copy(str, 1, 1) = "S"){
return string(v)
} else if(string_copy(str, 1, 1) = "U"){
return undefined
}
}
}
{ // Save GSON to file
function gsonRoomUnload(file){
global.roomUnload = file
with(all){
event_user(0)
}
event_user(0)
}
function saveString(fname, str){
var b = file_text_open_append(fname)
file_text_write_string(b, str)
file_text_close(b)
}
}
{ // Load GSON from file
function gsonRoomLoad(fname){
if(file_exists(fname)){
var file = file_text_open_read(fname)
var a = ""
var b = ""
for(var i = 0; ; i++){
b = file_text_readln(file)
a += b
if(string_copy(b, 1, 1) == "}"){
//gsonParseObject(a)
a = ""
}
if(string_copy(b, 2, 1) == ""){
break
}
}
return true
} else {
return false
}
}
}
function stringLineify(str){
var strings
strings[0] = ""
var a
a[1] = 0
a[2] = ""
for(var i = 0; a[2] != false; i++){
a = stringLine(str, a[1])
strings[i] = a[0]
}
return strings
}
function stringLine(str, index){
var carriage = "\n"
for(var i = 1; ; i++){
var b = string_copy(str, index + i, 1)
if(b == carriage){
return [string_copy(str, index, i), index + i + 1, true]
} else if(index + i >= string_length(str)){
return [string_copy(str, index, i), index + i + 1, false]
}
}
}
There are a couple of functions you'll notice aren't defined. Those functions are built into GMS. I've linked their documentation at the bottom of the question.
Okay, so here's my problem:
If I run gsonStringifySelf() then take the string it returns and put it in gsonParseObject() then this line:
string_copy(array[0], 2, string_length(array[0]) - 3)
Is correct. Remember, what this does is take this example string:
"§<type>{\n"
and turn it into this:
"<type>"
string_copy(array[0], 2, string_length(array[0]) - 3)
copies from the second location in the string (I will be using "|" to inducate where the code is selecting in the string)
"§|<type>{\n"
and string_length returns the length of the string. so its trying to copy within the "|" in the string
"§|<type>{\n|"
But it is actually trying to copy 1 past the final location of the string, because we skipped the first location of the string but told it to copy the length of the string. Thus, when we tell it how many spaces to copy, we must subtract 3. We subtract 1 so it is only copying up to the final point in the string, and another 2 so that "{\n" isn't included in the final string. (\n is newline, and is considered one character even though it is represented with 2)
so base string:
"§<type>{\n"
copy from second point
"§|<type>{\n"
copy to the end of the string +1
"§|<type>{\n|"
subtract one from how many we are copying to shave the extra space off the end
"§|<type>{\n|"
and then subtract another 2 from how many we are copying to remove "{\n"
"§|<type>|{\n"
and return it:
"<type>"
Here's the thing. If I run gsonStringifySelf() and put it in a file then when I pull it down from the file and put that into gsonParseObject(), instead of this being correct:
string_copy(array[0], 2, string_length(array[0]) - 3)
this is correct:
string_copy(array[0], 2, string_length(array[0]) - 4)
So somehow, in all the file beeswax, an extra character is being added on, and I can't figure out where.
Thanks in advance for the help.
bool(n)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Variable_Functions/bool.htm
real(n)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Strings/real.htm
is_bool(n)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Variable_Functions/is_bool.htm
is_numeric(n)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Variable_Functions/is_numeric.htm
is_string(n)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Variable_Functions/is_string.htm
is_undefined(n)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Variable_Functions/is_string.htm
asset_get_index(str)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Asset_Management/Assets_And_Tags/asset_get_index.htm
string_copy(str, index, count)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Strings/string_copy.htm
string_length(str)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Strings/string_length.htm
with(id)
https://manual.yoyogames.com/GameMaker_Language/GML_Overview/Language_Features/with.htm
event_user(int)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Asset_Management/Objects/Object_Events/event_user.htm
instance_create_depth()
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Asset_Management/Instances/instance_create_depth.htm
variable_instance_get()
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Variable_Functions/variable_instance_get.htm
variable_instance_set()
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/Variable_Functions/variable_instance_set.htm
file_exists(fname)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/File_Handling/File_System/file_exists.htm
file_text_open_append(fname)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/File_Handling/Text_Files/file_text_open_append.htm
file_text_write_string(file)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/File_Handling/Text_Files/file_text_write_string.htm
file_text_open_read(fname)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/File_Handling/Text_Files/file_text_open_read.htm
file_text_readln(file)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/File_Handling/Text_Files/file_text_readln.htm
file_text_close(file)
https://manual.yoyogames.com/GameMaker_Language/GML_Reference/File_Handling/Text_Files/file_text_close.htm
Ive tried most everything I can think of. The extra Character is an invisible character, and because of that, itis also invisible when I try to check the value with the debugger. I could bodge it, but if I can i'd rather avoid that, since it'll just cause me problems later anyway.

Answer 1

So after a bit of testing, I've realized that with just how long it takes to load a single instance, it's not worth using. Presumably I would use this for loading the areas during gameplay as well, but it takes well over 10 seconds to load a single instance, and a couple of minutes for more than 4 objects. I'll just have to avoid using arrays. Thanks! (bit annoying that I spent so much time on it for no reason tho...)

Answer 2

GMS is weird so arrays don't work with JSON formatting
The not-so-recently-added json_stringify does. The following
var thing = {
an_int: 1,
a_float: 1.5,
a_bool: true,
a_string: "hi",
an_array: [1, 2, "oh"],
a_struct: { x: 1, y: 2, name: "me", arr: [3, 4, undefined] },
};
show_debug_message(json_stringify(thing));
would output
{ "a_struct": { "x": 1.0, "y": 2.0, "name": "me", "arr": [ 3.0, 4.0, null ] }, "an_int": 1.0, "a_float": 1.5, "a_bool": true, "a_string": "hi", "an_array": [ 1.0, 2.0, "oh" ] }
With help of variable_instance_get_names/variable_struct_get_names, you could assemble a struct with all of the instance's variables (and desired built-in ones), encode that, and upon decoding write them back in.
As for your encoder-decoder, a few implementation caveats plague it:
GameMaker strings are immutable, meaning that adding two strings together generally allocates a new, third one. This makes your encoding slower than it could have been.
Writing to a buffer with kind=buffer_grow is a good way to limit memory re-allocations (as it will double the size whenever it runs out of space).
GameMaker strings are encoded as UTF-8, meaning that string_char_at isn't as simple as adding an index to the string pointer. This makes your decoding slower than it could have been.
Reading from a buffer is a common way around this, but take care - since you'd be storing a UTF-8 string in the buffer, a character will not necessarily be a single byte. Fortunately, you can usually completely ignore this detail so long as you read/write a string as a whole.
You are storing strings verbatim, which probably means that it's going to catch fire the moment one of the strings contains a } or other delimiter.
You are doing loops searching for characters often instead of calling string_pos[_ext] to let the engine do this [quicker].
Although technically recursive, its reliance on grabbing a substring with an encoded value instead of having a read position makes the process of reading nested values far messier than it should be (if I were to guess, it fails after misattributing one of the closing })
But also, you know, if it's your format, who said that it has to be a string? You can work with bytes instead, sparing yourself of a number of problems at once - no need for delimiters when you are writing/reading data in the same order.
A simple buffer-based encoder-decoder fits in just a little over 50 lines of code:
enum BinType { Undefined, Bool, Float, Int, String, Array, Struct };
function buffer_write_value(_buf, _val) {
if (is_real(_val)) {
buffer_write(_buf, buffer_u8, BinType.Float);
buffer_write(_buf, buffer_f64, _val);
} else if (is_bool(_val)) {
buffer_write(_buf, buffer_u8, BinType.Bool);
buffer_write(_buf, buffer_bool, _val);
} else if (is_numeric(_val)) {
buffer_write(_buf, buffer_u8, BinType.Int);
buffer_write(_buf, buffer_u64, _val); // u64 writes signed int64s fine
} else if (is_string(_val)) {
buffer_write(_buf, buffer_u8, BinType.String);
buffer_write(_buf, buffer_string, _val);
} else if (is_struct(_val)) {
buffer_write(_buf, buffer_u8, BinType.Struct);
var _names = variable_struct_get_names(_val);
var _count = array_length(_names);
buffer_write(_buf, buffer_u32, _count);
for (var i = 0; i < _count; i++) {
var _name = _names[i];
buffer_write(_buf, buffer_string, _name);
buffer_write_value(_buf, _val[$ _name]);
}
} else if (is_array(_val)) {
buffer_write(_buf, buffer_u8, BinType.Array);
var _count = array_length(_val);
buffer_write(_buf, buffer_u32, _count);
for (var i = 0; i < _count; i++) buffer_write_value(_buf, _val[i]);
} else buffer_write(_buf, buffer_u8, BinType.Undefined);
}
function buffer_read_value(_buf) {
switch (buffer_read(_buf, buffer_u8)) {
case BinType.Bool: return buffer_read(_buf, buffer_bool);
case BinType.Float: return buffer_read(_buf, buffer_f64);
case BinType.Int: return buffer_read(_buf, buffer_u64);
case BinType.String: return buffer_read(_buf, buffer_string);
case BinType.Array:
var _count = buffer_read(_buf, buffer_u32);
var _arr = array_create(_count);
for (var i = 0; i < _count; i++) _arr[i] = buffer_read_value(_buf);
return _arr;
case BinType.Struct:
var _struct = {};
repeat (buffer_read(_buf, buffer_u32)) {
var _name = buffer_read(_buf, buffer_string);
_struct[$ _name] = buffer_read_value(_buf);
}
return _struct;
default: return undefined;
}
}
and will successfully process the nested struct from the beginning of my answer if you write a value to a buffer, rewind it, and read it back. Add a little logic to detect instances or special cases, and you'll have a solution fit to your specific problems.
Further reading:
Haxe's JsonPrinter is a great example of a recursive JSON encoder.
SNAP is a GameMaker library with a number of encoders-decoders (following the aforementioned buffer principles, often non-recursive) that is both an example of good code and a solution to problems that you might be thinking of having.
Debugger (for stepping through and figuring out where you go wrong)
Profiler (for figuring out what's slow)
YYC (for performance)