I want to create a snowflake UDF which allocates some workdays based on the max and min dates.
Can we execute the javascript UDF like procedure?. Appreciate your thoughts on this
Getting an error while executing the below code:
CREATE OR REPLACE FUNCTION UDF_OTD_ADDWORKDAYS ("STARTDATE" date,"WORKDAYS" float,"FACTORYID" varchar(2))
RETURNS date
LANGUAGE Javascript
AS
$$
function dd(input)
{
var D=STARTDATE;
var WORKDAYS_REM=WORKDAYS;
var source_min_table =
`SELECT MIN(DateID) FROM CALENDAR;` ;
var min_day = snowflake.createStatement(
{sqlText: source_min_table});
var source_min_day = min_day.execute();
var source_max_table =
`SELECT max(DateID) FROM CALENDAR;` ;
var max_day = snowflake.createStatement(
{sqlText: source_max_table});
var source_max_day = max_day.execute();
if(WORKDAYS > 0)
{
while ((WORKDAYS_REM > 0) && (D < source_max_day))
{
var D = DATEADD(day, 1, D);
var addr_temp = snowflake.createStatement( { sqlText: "SELECT WorkDay_Int FROM CALENDAR WHERE FactoryID = '''+FACTORYID+''' AND DateID = '''+D''';" } );
var Adder = addr_temp.execute();
let WORKDAYS_REM=(WORKDAYS_REM-Adder);
}
return D;
}
if(WORKDAYS < 0)
{
var WORKDAYS_REM = ABS(WORKDAYS_REM)
while((WORKDAYS_REM > 0) && (D > source_min_day))
{
var D = DATEADD(day,-1, D);
var addr_temp = snowflake.createStatement( { sqlText: "SELECT WorkDay_Int FROM CALENDAR WHERE FactoryID = '''+FACTORYID+''' AND DateID = '''+D+''';" } );
var Adder = addr_temp.execute();
let WORKDAYS_REM = (WORKDAYS_REM-Adder);
}
return D;
}
return D;
}
return dd();
$$
;
error :
JavaScript execution error: Uncaught ReferenceError: snowflake is not defined in UDF_OTD_ADDWORKDAYS at 'var min_day = snowflake.createStatement(' position 14 stackstrace: dd line: 9 UDF_OTD_ADDWORKDAYS line: 44
JavaScript UDFs do not have the snowflake API object and they cannot run queries. What you could do if your CALENDAR table is small enough is send the whole table to your JavaScript function as a JSON. Here's an example using the Snowflake sample data:
select array_agg(object_construct(N.*)) as NATION_LIST from "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."NATION" N;
create or replace function USE_JSON_SCALAR(V variant)
returns string
language javascript
strict immutable
as
$$
return JSON.stringify(V[0].N_NAME);
$$;
select use_json_scalar(array_agg(object_construct(N.*))) as NATION_LIST from "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."NATION" N;
If there are other tables (most of the time there will be), just join in the CALENDAR or in this case the NATION table somewhere and build the JSON as shown.
create or replace procedure "CS_DEV"."META".SP_DATA_RETENTION("Trgt_DB" VARCHAR, "Trgt_Schema" VARCHAR, "Trgt_Stage" VARCHAR(20))
returns string
language JavaScript
EXECUTE AS CALLER
as
$$
var select_stmt = `SELECT * FROM ` + Trgt_DB + `.META.PREPROCESS_META`;
var select_stmt_sql = { sqlText: select_stmt };
var select_stmt_create = snowflake.createStatement(select_stmt_sql);
var select_stmt_exec = select_stmt_create.execute();`
while (select_stmt_exec.next()) {
var TGT_SCHEMA_NAME = select_stmt_exec.getColumnValue(5);
var TGT_TABLE_NAME = select_stmt_exec.getColumnValue(6);
var retentt = select_stmt_exec.getColumnValue(13);
var retentp = select_stmt_exec.getColumnValue(12);
var remove_data_tbl = `delete from ` +TGT_SCHEMA_NAME+ `.` +TGT_TABLE_NAME+ ` A USING META.PREPROCESS_META B where A.SYS_DATE <= DATEADD('`+retentt+`', -`+retentp+`, current_date()) AND B.RETENTION_TYPE = '`+retentt+`' AND B.RETENTION_PERIOD = `+retentp+ ` AND B.TGT_SCHEMA_NAME = '`+TGT_SCHEMA_NAME+ `' AND B.TGT_TABLE_NAME = '` +TGT_TABLE_NAME+`'`;
var remove_data_tbl_sql = {sqlText: remove_data_tbl };
var remove_data_tbl_create = snowflake.createStatement(remove_data_tbl_sql);
var remove_details_exec = remove_data_tbl_create.execute();}
Running the first statement produces the table above and then TGT_SCHEMA_NAME, TGT_TABLE_NAME, RETENTION_TYPE, and RETENTION_PERIOD values are taken and iterated through, however the procedure will only iterate over the first row DIM_AIRLINES.
Here's a slightly modified version. Can you run this and see its output? If it fails, look in the query history tab to see what SQL the procedure attempted to run. Then you can make adjustments as required. If you need more help post the error(s) you're getting when running the procedure and/or the SQL errors in the query history tab.
create or replace procedure "CS_DEV"."META".SP_DATA_RETENTION("Trgt_DB" VARCHAR, "Trgt_Schema" VARCHAR, "Trgt_Stage" VARCHAR(20))
returns string
language JavaScript
EXECUTE AS CALLER
as
$$
"option strict"
var sql = `SELECT * FROM ${Trgt_DB}.META.PREPROCESS_META`;
var rs = exec(sql);
var tgtSchemaName;
var tgtTableName;
var retentt;
var retentp;
var remove_data_tbl;
while (rs.next()) {
tgtSchemaName = select_stmt_exec.getColumnValue(5);
tgtTableName = select_stmt_exec.getColumnValue(6);
retentt = select_stmt_exec.getColumnValue(13);
retentp = select_stmt_exec.getColumnValue(12);
remove_data_tbl = `delete from ${tgtSchemaName}.${tgtTableName} A USING META.PREPROCESS_META B
where A.SYS_DATE <= DATEADD('$retentt', -${retentp}, current_date()) AND B.RETENTION_TYPE = '${retentt}' AND B.RETENTION_PERIOD = ${retentp} AND B.TGT_SCHEMA_NAME = '${tgtSchemaName}' AND B.TGT_TABLE_NAME = '${tgtTableName}'`;
exec(remove_data_tbl);
}
// ---- End main function. Start of helper functions.
function exec(sql){
cmd = {sqlText: sql};
stmt = snowflake.createStatement(cmd);
var rs;
rs = stmt.execute();
return rs;
}
$$;
I have a snowflake procedure using javascript.
the procedure exec the statement and return a single value result into a var:
var res = stmt.execute();
I want insert this value into a table. but this value is return as an object.
How do I work around the solution?
My script is dynamic and has a while loop that generates multiple statements, but here is a simplified version to focus on what I need to do get
Thx
create Procedure GetLastUpdateDate()
returns boolean
language javascript
as
$$
sql_command = "select max(Updatedate) from MyTable;";
var stmt = snowflake.createStatement({ sqlText: sql_command});
var res = stmt.execute();
snowflake.createStatement( { sqlText: 'INSERT INTO MYTABLE (NCOLUMN) VALUES (?)',
binds: [res]
} ).execute();
$$
;
.getColumnValue() could be used:
getColumnValue(colIdx|colName)
This method returns the value of a column in the current row (i.e. the row most recently retrieved by next()).
sql_command = "select max(Updatedate) from MyTable;";
var stmt = snowflake.createStatement({ sqlText: sql_command});
var res = stmt.execute();
while (res.next()) {
var myVal = res.getColumnValue(1);
snowflake.createStatement( { sqlText: 'INSERT INTO MYTABLE (NCOLUMN) VALUES (?)',
binds: [myVal]
} ).execute();
}
If more than one row is expected loop should be used: while (res.next()) {...}
I have a stored procedure that I just converted to Snowflake Javascript from PL/SQL. It inserts about 100 records a minute. The total record count is about 700. Because it is so difficult to know where a problem is in Snowflake, I insert log statements as the overall functionality progresses. I also push messages to an array that gets returned at the bottom. However, I do the insert into log table type things in PL/SQL and it barely makes a performance difference. I'll admit that my progress loading slows down the process, but am doubtful that it's the primary contributor.
The script makes a table that, given a date, shows the fiscal quarter that it corresponds to. This is helpful for other queries not shown. I have a simple loop that goes from the beginning of the first quarter to the end of the last and puts the corresponding quarter in the lookup table.
It took 9 minutes to run as written, but in Oracle, takes less than a second.
I'd like to know how to make this run faster:
create or replace procedure periodic_load()
RETURNS varchar
LANGUAGE javascript
execute as owner
as
$$
var result = "";
var messages = new Array();
try {
/**
Constants shared between functions
*/
var SINGLE_QUOTE_CHAR="'";
var DOUBLE_QUOTE_CHAR="\"";
var COMMA_CHAR=",";
var LEFT_PARENTHESIS="(";
var RIGHT_PARENTHESIS=")";
var ESCAPED_SINGLE_QUOTE_CHAR="\\'";
var ESCAPED_DOUBLE_QUOTE_CHAR="\\\"";
var CONSOLE_LOG_USED = true;
var IS_SNOWFLAKE = false;
/*
Execute Snowflake SQL or simulate the execution thereof
#parmam sqlTextIn,binds...
sqlTextIn: String of the sql command to run.
binds: zero or more parameters to bind to the execution of the command.
*/
function execute_with_log() {
var result = null;
messages.push('###'+"execute_with_log()");
messages.push('###'+"EXECUTE_WITH_LOG(BP1)");
var argumentsArray = Array.prototype.slice.apply(arguments);
var sqlTextIn = argumentsArray[0];
messages.push('###'+'EXECUTE_WITH_LOG argument count: '+arguments.length);
if(!IS_SNOWFLAKE) {
messages.push('###'+ "EXECUTE_WITH_LOG(BP2)");
console.log('SKIPPING SNOWFLAKE SQL: '+sqlTextIn);
} else {
messages.push('###'+ " EXECUTE_WITH_LOG(BP3)");
var statementResult;
var logMessage = sqlTextIn;
if(argumentsArray.length==1) {
messages.push('###'+ " EXECUTE_WITH_LOG(BP4)");
messages.push('###'+" ** NO BIND PARAMETERS DETECTED **");
} else {
messages.push('###'+ " EXECUTE_WITH_LOG(BP5)");
for(var bindParmCounter = 1; bindParmCounter < argumentsArray.length; bindParmCounter++) {
messages.push('###'+" ,"+argumentsArray[bindParmCounter]);
}
}
messages.push('###'+ " EXECUTE_WITH_LOG(BP6)");
log_message('I',logMessage);
if(argumentsArray.length===1) {
messages.push('###'+ " EXECUTE_WITH_LOG(BP7)");
statement = snowflake.createStatement( { sqlText: sqlTextIn });
} else {
messages.push('###'+ " EXECUTE_WITH_LOG(BP8)");
var bindsIn = argumentsArray.slice(1,argumentsArray.length);
for(var bindParmCounter = 0; bindParmCounter < bindsIn.length; bindParmCounter++) {
messages.push('###bindsIn['+bindParmCounter+"]="+bindsIn[bindParmCounter]);
messages.push('###bindsIn['+bindParmCounter+"] type ="+bindsIn[bindParmCounter].getName());
}
statement = snowflake.createStatement(
{
sqlText: sqlTextIn,
binds: bindsIn
}
);
}
messages.push('###'+ " EXECUTE_WITH_LOG(BP9) sqlTextIn="+sqlTextIn);
result = statement.execute();
messages.push('###'+ " After execute BP10 =");
commit();
messages.push('###'+ " After commit BP11 =");
}
return result;
}
function commit() {
messages.push('###'+ " commit");
statement = snowflake.createStatement(
{
sqlText: 'commit'
}
);
statement.execute();
return messages;
}
function log_message(severity,message) {
messages.push('###'+"log_message(severity,message): severity="+severity+" message="+message);
var result = null;
if(!IS_SNOWFLAKE) {
console.log(severity+": "+message);
messages.push('###'+severity+": "+message);
} else {
var record = {'severity': severity,'date_time': {value: 'current_timestamp::timestamp_ntz',useQuote:false},message:message};
try {
var escapeStep1=message.replaceAll(SINGLE_QUOTE_CHAR,ESCAPED_SINGLE_QUOTE_CHAR);
var escapeStep2=escapeStep1.replaceAll(DOUBLE_QUOTE_CHAR,ESCAPED_DOUBLE_QUOTE_CHAR);
quotedValue=SINGLE_QUOTE_CHAR+escapeStep2+SINGLE_QUOTE_CHAR;
var quotedSeverity = SINGLE_QUOTE_CHAR+severity+SINGLE_QUOTE_CHAR;
var sql_command = "insert into LOG_MESSAGES(severity,date_time,message) values("+quotedSeverity+",current_timestamp::timestamp_ntz,"+quotedValue+")";
statement = snowflake.createStatement( { sqlText: sql_command});
var sql_command = "commit";
statement = snowflake.createStatement( { sqlText: sql_command});
} catch(error) {
messages.push('###'+'FAILURE: '+error);
}
}
return result;
}
function truncate_table(tableName) {
messages.push('###'+"(truncate_table()");
var result = execute_with_log("truncate table "+tableName);
messages.push('###'+'I','End truncate_table()');
return result;
}
function fql() {
messages.push('###'+"begin fql()");
log_message('I','Begin fql()');
var table_name='fiscal_quarter_list';
truncate_table(table_name);
execute(
"insert into fiscal_quarter_list (fiscal_quarter_id,fiscal_quarter_name,fiscal_year,start_date,end_date,last_mod_date_stamp) ("
+" select fiscal_quarter_id,fiscal_quarter_name,fiscal_year,min(start_date) start_date,max(end_date) end_date,current_date from cdw_fiscal_periods cfp"
+" where (cfp.start_date >= add_months(sysdate(),-24) and sysdate() >= cfp.end_date ) or "
+" (cfp.start_date <= sysdate() and sysdate() < cfp.end_date) "
+" group by fiscal_quarter_id,fiscal_quarter_name,fiscal_year "
+" order by fiscal_quarter_id desc "
+" fetch first 8 rows only "
+")"
);
log_message('I','End fql()');
}
/*
Function to increment a Date object by one standard day
Sourced from https://stackoverflow.com/questions/563406/add-days-to-javascript-date
*/
function addDaysInJs(dateIn, days) {
var result = new Date(dateIn);
result.setDate(result.getDate() + days);
return result;
}
function dtfq() {
messages.push('###'+"dtfq()");
tableName = 'date_to_fiscal_quarter';
var firstDate;
var runningDate;
log_message('I','Begin dtfq');
truncate_table(tableName);
var result = null;
var resultSet = execute_with_log(" SELECT FISCAL_QUARTER_ID, FISCAL_QUARTER_NAME,try_to_date(START_DATE) as START_DATE, try_to_date(END_DATE) as END_DATE"
+ " FROM FISCAL_QUARTER_LIST "
+ " ORDER BY START_DATE ");
log_message('D','resultSet ='+resultSet);
log_message('D','resultSet typeof='+typeof resultSet);
while(resultSet.next()) {
messages.push('###'+"bp1 dtfq() loop start_date="+resultSet.getColumnValue("START_DATE")+" end_date="+resultSet.getColumnValue("END_DATE"));
firstDate = resultSet.getColumnValue("START_DATE");
lastDate = resultSet.getColumnValue("END_DATE");
runningDate=new Date(firstDate);
lastDate = new Date(lastDate);
log_message('D','Start date='+firstDate);
while (runningDate <= lastDate) {
var fiscalQuarterId=resultSet.getColumnValue("FISCAL_QUARTER_ID")
var fiscalQuarterName=resultSet.getColumnValue("FISCAL_QUARTER_NAME")
messages.push('###'+"bp2 dtfq() runningDate="+runningDate+' fiscalQuarterId='+fiscalQuarterId+' fiscalQuarterName='+fiscalQuarterName);
log_message('D','Fiscal quarter id='+fiscalQuarterId);
/*
execute_with_log(" insert into sc_hub_date_to_fiscal_quarter(date_stamp,) "
+" values(try_to_date(?)) "
,runningDate.toISOString());
*/
execute_with_log(" insert into sc_hub_date_to_fiscal_quarter(date_stamp,fiscal_quarter_id,fiscal_quarter_name) "
+" values(?,?,?)"
,runningDate.toISOString()
,fiscalQuarterId
,fiscalQuarterName);
runningDate = addDaysInJs(runningDate, 1);
}
}
log_message('I','End dtfq Success');
return result;
}
/*
Execute Snowflake SQL or simulate the execution thereof
#parmam sqlTextIn,binds...
sqlTextIn: String of the sql command to run.
binds: zero or more parameters to bind to the execution of the command.
*/
function execute() {
messages.push('###'+"execute():");
var result = null;
var argumentsArray = Array.prototype.slice.apply(arguments);
var sqlTextIn = argumentsArray[0];
if(!IS_SNOWFLAKE) {
console.log('SKIPPING SNOWFLAKE SQL: '+sqlTextIn);
messages.push('###'+'SKIPPING SNOWFLAKE SQL: '+sqlTextIn);
} else {
messages.push('###'+'USING SNOWFLAKE SQL: '+sqlTextIn);
var statementResult;
if(argumentsArray.length>2) {
messages.push('###'+'Has bind arguments: ');
var bindsIn = argumentsArray.slice(2,argumentsArray.length);
statement = snowflake.createStatement(
{
sqlText: sqlTextIn,
binds: bindsIn
}
);
} else {
messages.push('###'+'Has no bind arguments: ');
messages.push('###'+'###sqlText='+sqlTextIn+'###');
statement = snowflake.createStatement( { sqlText: sqlTextIn });
}
result = statement.execute();
messages.push('###'+'statement.execute succeeded');
log_message('I',sqlTextIn);
}
return result;
}
String.prototype.replaceAll = function(target, replacement) {
return this.split(target).join(replacement);
};
Object.prototype.getName = function() {
var funcNameRegex = /function (.{1,})\(/;
var results = (funcNameRegex).exec((this).constructor.toString());
return (results && results.length > 1) ? results[1] : "";
};
dtfq();
} catch(error) {
messages.push('###'+error);
} finally {
result = messages.join("\n");
}
return result;
$$
;
call periodic_load()
The use-case isn't entirely stated here, but it appears that your stored procedure merely generates (explodes) and inserts a series of dates into a table, for each date range encountered in a source table input row.
This can be achieved with SQL (with recursive CTEs) directly, which would run far more efficiently than a linear stored procedure iteration:
create table destination_table (fiscal_quarter_id integer, fiscal_quarter_name string, date_stamp date);
insert into destination_table
with source_table(fiscal_quarter_id, fiscal_quarter_name, start_date, end_date) as (
select 1, 'Q1', '2020-01-01'::date, '2020-03-31'::date union all
select 2, 'Q2', '2020-04-01'::date, '2020-06-30'::date union all
select 3, 'Q3', '2020-07-01'::date, '2020-09-30'::date union all
select 4, 'Q4', '2020-10-01'::date, '2020-12-31'::date
), recursive_expand as (
select
fiscal_quarter_id, fiscal_quarter_name, start_date, end_date,
start_date as date_stamp
from source_table
union all
select
fiscal_quarter_id, fiscal_quarter_name, start_date, end_date,
dateadd(day, 1, date_stamp)::date date_stamp
from recursive_expand
where date_stamp < end_date
)
select fiscal_quarter_id, fiscal_quarter_name, date_stamp
from recursive_expand
order by date_stamp asc;
The example inserts 366 rows into the destination_table (2020 being a leap year) covering dates of all four quarters.
#Greg Pavlik's comment covers why the stored procedure is slow due to executing whole statements (each independently submitted, compiled, planned, executed, and returned from the snowflake query processing service adds a lot of overhead). If you'd still like to proceed with the stored procedures API for your use-case, an idea is to make two specific changes:
Store all generated data rows into an array instead of inserting them directly, like so (this is only practical for a few hundred rows, not beyond, due to memory constraints):
function dtfq() {
var all_rows = [];
// … iteration and other logic here …
all_rows.push([fiscalQuarterId, fiscalQuarterName, runningDate]);
// … iteration and other logic ends here (minus inserts) …
return all_rows;
}
Insert the list of n rows generated using a single generated INSERT statement with n value containers. An example of such code can be seen in this answer.
I am trying to push key, value in array and then converting it into JSON using JSON.stringify(). But it is not working.
My node.js code:
var jarray=[];
var json1="";
for (var i=0; i<jsonObj["Masters"]['Customer'].length; i++){
var name= jsonObj["Masters"]['Customer'][i];
var cust_name=name['Customer_Name'];
var cust_code=name['Customer_Code'];
connection.query("SELECT code FROM ((SELECT ccode AS code FROM customermaster WHERE companyid='AXWPM1658D') UNION ALL (SELECT scode AS code FROM suppliermaster WHERE companyid='AXWPM1658D') UNION ALL (SELECT stcode AS code FROM stockmaster WHERE companyid='AXWPM1658D') UNION ALL (SELECT gcode AS code FROM generalledger2 WHERE companyid='AXWPM1658D') UNION ALL (SELECT bcode AS code FROM bankmaster WHERE companyid='AXWPM1658D'))p where code='"+cust_code+"' ", function(err, rows, fields) {
if (!err){
var item ={"customer_name":cust_name ,"customer_code": cust_code };
jarray.push(item);
}
else{
console.log('Error while performing Query.'+err);
}
});
}
json1=JSON.stringify({jarray:jarray});
var jsonObj1 = JSON.parse(json1);
console.log("Json:"+jsonObj1);
console.log("arr length:"+jsonObj1.jarray.length);
It prints:
Json:{ jarray: [] }
arr length:0
My question is how to push values in array and convert it into JSON array?
What do you use to make SQL requests? It looks like it has an asynchronous behaviour. If so, it can be the reason on why your array is still empty when you stringify it. By the way, making SQL request in a loop is not very effective. Perhaps is it better to fetch all results with one SQL request, this will also make it easier to stringify in the callback
Use this:
var jarray = [];
var json1 = "";
var async = require('async');
async.forEachLimit(jsonObj["Masters"]['Customer'], 1, function(customer, callback) {
var name = customer;
var cust_name = name['Customer_Name'];
var cust_code = name['Customer_Code'];
connection.query("SELECT code FROM ((SELECT ccode AS code FROM customermaster WHERE companyid='AXWPM1658D') UNION ALL (SELECT scode AS code FROM suppliermaster WHERE companyid='AXWPM1658D') UNION ALL (SELECT stcode AS code FROM stockmaster WHERE companyid='AXWPM1658D') UNION ALL (SELECT gcode AS code FROM generalledger2 WHERE companyid='AXWPM1658D') UNION ALL (SELECT bcode AS code FROM bankmaster WHERE companyid='AXWPM1658D'))p where code='" + cust_code + "' ", function(err, rows, fields) {
if (!err) {
var item = {
"customer_name": cust_name,
"customer_code": cust_code
};
jarray.push(item);
callback();
} else {
callback(err);
console.log('Error while performing Query.' + err);
}
});
}, function(err) {
if (err) {
console.log(err);
} else {
json1 = JSON.stringify({
jarray: jarray
});
var jsonObj1 = JSON.parse(json1);
console.log("Json:" + jsonObj1);
console.log("arr length:" + jsonObj1.jarray.length);
}
})