App script/maker and Big Query - arrays

I having an google app (app maker) where I script the following code:
function runQuery() {
var projectId = 'projekte-123425512';
var request = {
query: 'SELECT title FROM [bigquery-public-data:samples.wikipedia] where title contains "olimpic" LIMIT 100'
};
var queryResults = BigQuery.Jobs.query(request, projectId);
var jobId = queryResults.jobReference.jobId;
var names = queryResults.schema.fields.map(function(field){ return field.name; });
return queryResults.rows.map(function(row) {
var obj = {};
for( var i = 0, len = names.length; i < len; ++i ) {
obj[names[i]] = row.f[i].v;
}
return obj;
});
}
So I basically try to get some data out of Big Query and store it in an array. Later I want it as a calculated model and a data source in my app:
I already tried it and get result in my logger:
It works! But only in app script debugger, then I want to test the whole app I get the following error:
The function queryRecords must return an array of records, but the array contained an element that was not a record. Error: The function queryRecords must return an array of records, but the array contained an element that was not a record.
EDIT
I updated my code
function runQuery() {
var projectId = 'nifty-stage-155512';
var request = {
query: 'SELECT title FROM [bigquery-public-data:samples.wikipedia] where title contains "olimpic" LIMIT 100'
};
var queryResults = BigQuery.Jobs.query(request, projectId);
var jobId = queryResults.jobReference.jobId;
var names = queryResults.schema.fields.map(function(field){ return field.name; });
//var records = [];
return queryResults.rows.map(function(row) {
var record = app.models.Test.newRecord();
for (var i = 0, len = names.length; i < len; ++i) {
// Calculated model should contain correspondent fields
// all non-defined fields will be ignored
record[names[i]] = (row.f[i].v);
}
return record;
});
}
It now works without error but I still get no data into my grid:
Is there something I'am missing in the configuration of the grid or the datasource??

App Maker doesn't allow to return arbitrary objects through its datasources. All results should be strongly typed:
...
return queryResults.rows.map(function(row) {
var record = app.models.CalcModelName.newRecord();
for (var i = 0, len = names.length; i < len; ++i) {
// Calculated model should contain correspondent fields
// all non-defined fields will be ignored
record[names[i]] = row.f[i].v;
}
return record;
});
Here are some samples for reference
https://developers.google.com/appmaker/samples/calculated-model/
https://developers.google.com/appmaker/samples/jdbc/

Related

Filter one array using a 2nd array, and display the results on a spreadsheet

----- BACKGROUND ----- In Google Sheets/Apps Script, I'm trying to move some transaction data from one sheet and organize it on a different sheet with subtotal rows for each customer account (called "Units") with a grand total at the bottom.
I have the transactions organized in an array of objects as key:value pairs ("transactions", the keys are headers, some values are strings, others are numbers), and I have the units in a 2nd array called "unitList". I want to iterate through each element in the unitList array, and pull each transaction with a matching unit onto my "targetSheet", then append a subtotal row for each unit.
----- UPDATE 10/7/2018 3:49PM EST -----
Thanks to everyone for your input - I took your advice and ditched the library I was using and instead found better getRowsData and appendRowsData functions which I put directly in my code project. This fixed the array filter problem (verified by logging filterResults), BUT, now when I call appendRowsData(), I get this error:
The coordinates or dimensions of the range are invalid. (line 73, file "Display Transactions")
Line 73 is the line below, in the appendRowsData function. Any help on how to fix this would be greatly appreciated.
var destinationRange = sheet.getRange(firstDataRowIndex, 1, objects.length, 9);
Here's my project in it's entirety thus far:
function displayTransactions() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
// Label each sheet
var dashboard = ss.getSheetByName('Dashboard');
var unitsSheet = ss.getSheetByName('Unit Ref Table');
var transactionsSheet = ss.getSheetByName('Transactions Ref Sheet');
var targetSheet = ss.getSheetByName('Target Sheet');
// Returns true if the cell where cellData was read from is empty.
// Arguments:
// - cellData: string
function isCellEmpty(cellData) {
return typeof(cellData) == "string" && cellData == "";
}
// Define function that converts arrays into JSON
// For every row of data in data, generates an object that contains the data.
// Names of object fields are defined in keys.
// Arguments:
// - data: JavaScript 2d array
// - keys: Array of Strings that define the property names for the objects to create
function getObjects(data, keys) {
var objects = [];
for (var i = 0; i < data.length; ++i) {
var object = {};
var hasData = false;
for (var j = 0; j < data[i].length; ++j) {
var cellData = data[i][j];
if (isCellEmpty(cellData)) {
continue;
}
object[keys[j]] = cellData;
hasData = true;
}
if (hasData) {
objects.push(object);
}
}
return objects;
}
// Define function that pulls spreadsheet data into arrays, then converts to JSON using getObjects function
function getRowsData(sheet) {
var headersRange = sheet.getRange(1, 1, 1, sheet.getLastColumn());
var headers = headersRange.getValues()[0];
var dataRange = sheet.getRange(sheet.getFrozenRows()+1, 1, sheet.getLastRow(), sheet.getLastColumn());
return getObjects(dataRange.getValues(), headers);
}
// Define function appendRowsData that uses getLastRow to fill in one row of data per object defined in the objects array.
// For every Column, it checks if data objects define a value for it.
// Arguments:
// - sheet: the sheet object where the data will be written
// - objects: an array of objects, each of which contains data for a row
function appendRowsData(sheet, objects) {
var headersRange = sheet.getRange(7, 1, 1, 9);
var firstDataRowIndex = sheet.getLastRow() + 1;
var headers = headersRange.getValues()[0];
var data = [];
for (var i = 0; i < objects.length; ++i) {
var values = []
for (j = 0; j < headers.length; ++j) {
var header = headers[j];
values.push(header.length > 0 && objects[i][header] ? objects[i][header] : "");
}
data.push(values);
}
var destinationRange = sheet.getRange(firstDataRowIndex, 1, objects.length, 9);
destinationRange.setValues(data);
}
// Call getRowsData on transactions sheet
var transactions = getRowsData(transactionsSheet);
// Get array of units
var unitList = unitsSheet.getRange("B2:B").getValues();
// Iterate through the unitList and pull all transactions with matching unit into the target sheet
for (var i=0; i < unitList.length; i++) {
var subTotal = 0;
var grandTotal = 0;
var filterResults = transactions.filter(function(x) {
return x['Unit'] == unitList[i];
})
Logger.log(filterResults); // This brings the correct results!
// Display matching transactions
appendRowsData(targetSheet, filterResults);
// Grand total at the bottom when i=units.length
}
}

Google script, empty array from fetched API

I'm trying to fetch my ETH balance and transactions from etherscan.io website and I'm trying to use the same code I used before for another website. But it seems returning me an empty array, and also the error "The coordinates or dimensions of the range are invalid."
This is the code:
function getTx() {
var sheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Data");
var url = 'http://api.etherscan.io/api?module=account&action=txlist&address=0x49E81AA0cFE7eeA9738430212DC6677acF2f01a1&sort=asc';
var json = UrlFetchApp.fetch(url).getContentText();
var data = JSON.parse(json);
var rows = [],
array;
for (i = 0; i < data.length; i++) {
array = data[i];
rows.push([array.timeStamp, array.from, array.to, array.value]);
}
Logger.log(rows)
askRange = sheet.getRange(3, 1, rows.length, 3);
askRange.setValues(rows);
}
The logged "rows" is empty, what am I doing wrong?
Thank you
How about a following modification?
Modification points :
There is data you want at data.result.
Number of columns of data is 4.
Modified script :
function getTx() {
var sheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName("Data");
var url = 'http://api.etherscan.io/api?module=account&action=txlist&address=0x49E81AA0cFE7eeA9738430212DC6677acF2f01a1&sort=asc';
var json = UrlFetchApp.fetch(url).getContentText();
var data = JSON.parse(json);
var rows = [],
array;
for (i = 0; i < data.result.length; i++) { // Modified
array = data.result[i];
rows.push([array.timeStamp, array.from, array.to, array.value]);
}
Logger.log(rows)
askRange = sheet.getRange(3, 1, rows.length, 4); // Modified
askRange.setValues(rows);
}
If I misunderstand your question, I'm sorry.

How to merge REST call results in Angular app more efficiently

I have an Angular SPA running on a SharePoint 2013 page. In the code, I'm using $q to pull data from 10 different SharePoint lists using REST and then merging them into one JSON object for use in a grid. The code runs and outputs the intended merged data but it's leaky and crashes the browser after a while.
Here's the code in the service:
factory.getGridInfo = function() {
var deferred = $q.defer();
var list_1a = CRUDFactory.getListItems("ListA", "column1,column2,column3");
var list_1b = CRUDFactory.getListItems("ListB", "column1,column2,column3");
var list_2a = CRUDFactory.getListItems("ListC", "column4");
var list_2b = CRUDFactory.getListItems("ListD", "column4");
var list_3a = CRUDFactory.getListItems("ListE", "column5");
var list_3b = CRUDFactory.getListItems("ListF", "column5");
var list_4a = CRUDFactory.getListItems("ListG", "column6");
var list_4b = CRUDFactory.getListItems("ListH", "column6");
var list_5a = CRUDFactory.getListItems("ListI", "column7");
var list_5b = CRUDFactory.getListItems("ListJ", "column7");
$q.all([list_1a, list_1b, list_2a, list_2b, list_3a, list_3b, list_4a, list_4b, list_5a, list_5b])
.then(function(results){
var results_1a = results[0].data.d.results;
var results_1b = results[1].data.d.results;
var results_2a = results[2].data.d.results;
var results_2b = results[3].data.d.results;
var results_3a = results[4].data.d.results;
var results_3b = results[5].data.d.results;
var results_4a = results[6].data.d.results;
var results_4b = results[7].data.d.results;
var results_5a = results[8].data.d.results;
var results_5b = results[9].data.d.results;
var combined_1 = results_1a.concat(results_1b);
var combined_2 = results_2a.concat(results_2b);
var combined_3 = results_3a.concat(results_3b);
var combined_4 = results_4a.concat(results_4b);
var combined_5 = results_5a.concat(results_5b);
for(var i = 0; i < combined_1.length; i++){
var currObj = combined_1[i];
currObj["column4"] = combined_2[i].column4;
currObj["column5"] = combined_3[i].column5;
currObj["column6"] = combined_4[i].column6;
currObj["column7"] = combined_5[i].column7;
factory.newObjectArray[i] = currObj;
}
deferred.resolve(factory.newObjectArray);
},
function (error) {
deferred.reject(error);
});
return deferred.promise;
};
Here's the REST call in CRUDFactory:
factory.getListItems = function (listName, columns){
var webUrl = _spPageContextInfo.webAbsoluteUrl + "/_api/web/lists/getByTitle('"+listName+"')/items?$select="+columns+"&$top=5000";
var options = {
headers: { "Accept": "application/json; odata=verbose" },
method: 'GET',
url: webUrl
};
return $http(options);
};
And then here's the controller bit:
$scope.refreshGridData = function(){
$scope.hideLoadingGif = false;
$scope.GridData = "";
GlobalFactory.getGridInfo()
.then(function(){
$scope.GridData = GlobalFactory.newObjectArray;
$scope.hideLoadingGif = true;
});
};
UPDATE 1: Per request, Here's the HTML (just a simple div that we're using angular-ui-grid on)
<div ui-grid="GridOptions" class="grid" ui-grid-selection ui-grid-exporter ui-grid-save-state></div>
This code starts by declaring some get calls and then uses $q.all to iterate over the calls and get the data. It then stores the results and merges them down to 5 total arrays. Then, because my list structure is proper and static, I'm able to iterate over one of the merged arrays and pull data from the other arrays into one master array that I'm assigning to factory.newObjectArray, which I'm declaring as a global in my service and using as my grid data source.
The code runs and doesn't kick any errors up but the issue is with (I believe) the "getGridInfo" function. If I don't comment out any of the REST calls, the browser uses 45 MB of data that doesn't get picked up by GC which is then compounded for each click until the session is ended or crashes. If I comment out all the calls but one, my page only uses 18.4 MB of memory, which is high but I can live with it.
So what's the deal? Do I need to destroy something somewhere? If so, what and how? Or does this relate back to the REST function I'm using?
UPDATE 2: The return result that the grid is using (the factory.newObjectArray) contains a total of 5,450 items and each item has about 80 properties after the merge. The code above is simplified and shows the pulling of a couple columns per list, but in actuality, I'm pulling 5-10 columns per list.
At the end of the day you are dealing with a lot of data, so memory problems are potentially always going to be an issue and you should probably consider whether you need to have all the data in memory.
The main goal you should probably be trying to achieve is limiting duplication of arrays, and trying to keep the memory footprint as low as possible, and freeing memory as fast as possible when you're done processing.
Please consider the following. You mention the actual number of columns being returned are more than your example so I have taken that into account.
factory.getGridInfo = function () {
var deferred = $q.defer(),
// list definitions
lists = [
{ name: 'ListA', columns: ['column1', 'column2', 'column3'] },
{ name: 'ListB', columns: ['column1', 'column2', 'column3'], combineWith: 'ListA' },
{ name: 'ListC', columns: ['column4'] },
{ name: 'ListD', columns: ['column4'], combineWith: 'ListC' },
{ name: 'ListE', columns: ['column5'] },
{ name: 'ListF', columns: ['column5'], combineWith: 'ListE' },
{ name: 'ListG', columns: ['column6'] },
{ name: 'ListH', columns: ['column6'], combineWith: 'ListG' },
{ name: 'ListI', columns: ['column7'] },
{ name: 'ListJ', columns: ['column7'], combineWith: 'ListI' },
],
// Combines two arrays without creating a new array, mindful of lenth limitations
combineArrays = function (a, b) {
var len = b.length;
for (var i = 0; i < len; i = i + 5000) {
a.unshift.apply(a, b.slice(i, i + 5000));
}
};
$q.all(lists.map(function (list) { return CRUDFactory.getListItems(list.name, list.columns.join()); }))
.then(function (results) {
var listResultMap = {}, var baseList = 'ListA';
// map our results to our list names
for(var i = 0; i < results.length; i++) {
listResultMap[lists[i].name] = results[i].data.d.results;
}
// loop around our lists
for(var i = 0; i < lists.length; i++) {
var listName = lists[i].name, combineWith = lists[i].combineWith;
if(combineWith) {
combineArrays(listResultMap[combineWith], listResultMap[listName]);
delete listResultMap[listName];
}
}
// build result
factory.newObjectArray = listResultMap[baseList].map(function(item) {
for(var i = 0; i < lists.length; i++) {
if(list.name !== baseList) {
for(var c = 0; c < lists[i].columns.length; c++) {
var columnName = lists[i].columns[c];
item[columnName] = listResultMap[list.name][columnName];
}
}
}
return item;
});
// clean up our remaining results
for (var i = 0; i < results.length; i++) {
delete results[i].data.d.results;
delete results[i];
}
deferred.resolve(factory.newObjectArray);
},
function (error) {
deferred.reject(error);
});
return deferred.promise;
};
I would suggest to add some sort of paging option... It's perhaps not a great idea to add all results to one big list.
Next i would suggest against ng-repeat or add a "track by" to the repeat function.
Check out: http://www.alexkras.com/11-tips-to-improve-angularjs-performance/
Fiddler your queries, the issue is probably rendering all the elements in the dom... Which could be kinda slow ( investigate)

Array retrieval in HTML5 with the help of LocalStorage

I have 1 entry in my local storage file
10/10/2014 19:23:34 (date and time) as a key and
{"entries":[{"name":"s","contact":"s","location":"s","email":"sweety#yahoo.com","detail":"s","f_name":"form1"}]}
(form field records) as value
so my question is how to get this array with separated parameters like Name = s, Contact = s, etc...
If you didn't already parse your data you can parse it like this:
var json = localStorage.getItem('10/10/2014 19:23:34');
if (json) {
var data = JSON.parse(json);
if (data !== null) {
// then you can try to do something with the data within it
var entries = data.entries;
for (var i = 0, l = entries.length; i < l; i++) {
var entry = entries[i];
console.log(entry.name, entry.contact, entry.location, entry.email);
}
}
}

How to loop range of cells and set value in adjacent column

I'm learning Google Apps Scripts for use with Google Spreadsheets.
I have a list of URLs in one column and I want to write a script to get the title element from each URL and write it in the adjacent cell. I have accomplished this for one specific cell as per the following script:
function getTitles() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName("url_list");
var range = sheet.getRange("G3");
var url = range.getValue();
var response = UrlFetchApp.fetch(url);
var doc = Xml.parse(response.getContentText(),true);
var title = doc.html.head.title.getText();
var output = sheet.getRange("H3").setValue(title);
Logger.log(title);
return title;
}
This gets the URL in G3, parses it, pulls the element and writes the output in H3.
Now that I have this basic building block I want to loop the entire G column and write the output to the adjacent cell but I'm stuck. Can anyone point me in the right direction?
May look something like this:
function getTitles() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName("url_list");
var urls = sheet.getRange("G3:G").getValues();
var titleList = [], newValues = [],
response, doc, title;
for (var row = 0, var len = urls.length; row < len; row++) {
if (urls[row] != '') {
response = UrlFetchApp.fetch(urls[row]);
doc = Xml.parse(response.getContentText(),true);
title = doc.html.head.title.getText();
newValues.push([title]);
titleList.push(title);
Logger.log(title);
} else newValues.push([]);
}
Logger.log('newValues ' + newValues);
Logger.log('titleList ' + titleList);
// SET NEW COLUMN VALUES ALL AT ONCE!
sheet.getRange("H3").offset(0, 0, newValues.length).setValues(newValues);
return titleList;
}

Resources