I am trying to create and execute a simple Snowflake stored procedure which takes an input argument and creates a stage. However, when I try to call the procedure it throws
ERROR: invalid value [:] for parameter
create or replace procedure raw.test.create_stage_test(PARM_URL string)
returns string
language javascript
execute as owner
as
$$
var cmd = `create or replace stage raw.test.agency_type_test
url =:1
file_format = (type = json)
credentials = (aws_role = 'arn:aws:iam::myrole');`
var sql_statement = snowflake.createStatement({
sqlText: cmd,
binds: [PARM_URL]
});
try {
var rs = sql_statement.execute();
rs.next()
rcount = rs.getColumnValue(1);
if (rcount == 0){
throw "row count is 0";
}
return "Rows count: " + rcount;
}
catch (err) {
return "Failed: " + err; // Return a success/error indicator.
}
$$;
CALL raw.test.create_stage_test('s3://mybucket');
Perhaps using interpolation, as in the following code snippet, could be an alternative? Note the single quotes included around the url value:
var cmd = `create or replace stage agency_type_test
url='${PARM_URL}'
file_format = (type = json)
credentials = (aws_role = 'arn:aws:iam::myrole');`
var sql_statement = snowflake.createStatement({
sqlText: cmd
});
Related
I have created a stored procedure to accept 2 parameters which helps to determine load type and based on that I am updating the control table to change dates.
When I am calling the SP it is failing due to:
Unexpected identifier in USP_NIGHTLYJOBRESETDAYS at
' ResetDaysDateRange = SELECT (`DATEADD(day,-?,DATEADD(day,DATEDIFF(day, '1900-01-01'::date, CURRENT_TIMESTAMP::date),'1900-01-01'::date))`,binds:[RESETDAYS]);' position 33
Here is the stored procedure code
CREATE OR REPLACE PROCEDURE etl.usp_NightlyJobResetDays (NIGHTLYLOAD VARCHAR(10), RESETDAYS VARCHAR(10))
RETURNS VARCHAR
LANGUAGE JAVASCRIPT
EXECUTE AS CALLER
AS
$$
var sql_command =
`BEGIN
let ResetDaysDateRange;
//Capturing date based on value
ResetDaysDateRange = SELECT (`DATEADD(day,-?,DATEADD(day,DATEDIFF(day, '1900-01-01'::date, CURRENT_TIMESTAMP::date),'1900-01-01'::date))`,binds:[RESETDAYS]);
//checking load type
if (NIGHTLYLOAD ='Yes')
{
EXEC(`Update Reporting.ReportingLoadDetail
SET MaxLoadDate = ?
WHERE IsNightlyLoadImpacted <> 'Yes'`,[ResetDaysDateRange]);
EXEC(`UPDATE Reporting.ReportingLoadDetail
SET MaxLoadDate = CASE WHEN ? > LastInitialLoadDate THEN ?
ELSE LastInitialLoadDate
END
WHERE IsNightlyLoadImpacted = 'Yes'`,[ResetDaysDateRange,ResetDaysDateRange]);
}
//reset restartabilityStatus to completed if last incremental load got failed
EXEC(`UPDATE etl.APILastLoadDetail set RestartabilityStatus = 'Completed'`);
END`
try {
snowflake.execute (
{sqlText: sql_command}
);
return "Succeeded."; // Return a success/error indicator.
}
catch (err) {
return "Failed: " + err; // Return a success/error indicator.
}
$$
;
//Calling sp
call etl.usp_NightlyJobResetDays('Yes',30);
something like this should work:
CREATE OR REPLACE PROCEDURE ETL.usp_NightlyJobResetDays (NIGHTLYLOAD VARCHAR(10), RESETDAYS VARCHAR(10))
RETURNS VARCHAR
LANGUAGE JAVASCRIPT
EXECUTE AS CALLER
AS
$$
try{
//Capturing date based on value
var sql_command1 = `SELECT TO_CHAR((DATEADD(day,-`+RESETDAYS+`,DATEADD(day,DATEDIFF(day, '1900-01-01'::date, CURRENT_TIMESTAMP::date),'1900-01-01'::date))))`;
var ResetDaysDateRange_res = snowflake.execute ({sqlText: sql_command1});
ResetDaysDateRange_res.next()
var ResetDaysDateRange = ResetDaysDateRange_res.getColumnValue(1);
//checking load type
if (NIGHTLYLOAD == `Yes`)
{
var sql_command2 = `Update Reporting.ReportingLoadDetail
SET MaxLoadDate = TO_DATE('`+ResetDaysDateRange+`')
WHERE IsNightlyLoadImpacted <> 'Yes'`;
snowflake.execute ({sqlText: sql_command2});
var sql_command3 = `UPDATE Reporting.ReportingLoadDetail
SET MaxLoadDate = CASE WHEN TO_DATE('`+ResetDaysDateRange+`') > LastInitialLoadDate THEN TO_DATE('`+ResetDaysDateRange+`')
ELSE LastInitialLoadDate
END
WHERE IsNightlyLoadImpacted = 'Yes'`;
snowflake.execute ({sqlText: sql_command3});
}
//reset restartabilityStatus to completed if last incremental load got failed
var sql_command4 = `UPDATE etl.APILastLoadDetail set RestartabilityStatus = 'Completed'`;
snowflake.execute ({sqlText: sql_command4});
return "Success";
}
catch (err) {
return "Failed: " + err; // Return a success/error indicator.
}
$$
;
Best regards,
TK
create or replace procedure "CS_DEV"."META".SP_DATA_RETENTION("Trgt_DB" VARCHAR, "Trgt_Schema" VARCHAR, "Trgt_Stage" VARCHAR(20))
returns string
language JavaScript
EXECUTE AS CALLER
as
$$
var select_stmt = `SELECT * FROM ` + Trgt_DB + `.META.PREPROCESS_META`;
var select_stmt_sql = { sqlText: select_stmt };
var select_stmt_create = snowflake.createStatement(select_stmt_sql);
var select_stmt_exec = select_stmt_create.execute();`
while (select_stmt_exec.next()) {
var TGT_SCHEMA_NAME = select_stmt_exec.getColumnValue(5);
var TGT_TABLE_NAME = select_stmt_exec.getColumnValue(6);
var retentt = select_stmt_exec.getColumnValue(13);
var retentp = select_stmt_exec.getColumnValue(12);
var remove_data_tbl = `delete from ` +TGT_SCHEMA_NAME+ `.` +TGT_TABLE_NAME+ ` A USING META.PREPROCESS_META B where A.SYS_DATE <= DATEADD('`+retentt+`', -`+retentp+`, current_date()) AND B.RETENTION_TYPE = '`+retentt+`' AND B.RETENTION_PERIOD = `+retentp+ ` AND B.TGT_SCHEMA_NAME = '`+TGT_SCHEMA_NAME+ `' AND B.TGT_TABLE_NAME = '` +TGT_TABLE_NAME+`'`;
var remove_data_tbl_sql = {sqlText: remove_data_tbl };
var remove_data_tbl_create = snowflake.createStatement(remove_data_tbl_sql);
var remove_details_exec = remove_data_tbl_create.execute();}
Running the first statement produces the table above and then TGT_SCHEMA_NAME, TGT_TABLE_NAME, RETENTION_TYPE, and RETENTION_PERIOD values are taken and iterated through, however the procedure will only iterate over the first row DIM_AIRLINES.
Here's a slightly modified version. Can you run this and see its output? If it fails, look in the query history tab to see what SQL the procedure attempted to run. Then you can make adjustments as required. If you need more help post the error(s) you're getting when running the procedure and/or the SQL errors in the query history tab.
create or replace procedure "CS_DEV"."META".SP_DATA_RETENTION("Trgt_DB" VARCHAR, "Trgt_Schema" VARCHAR, "Trgt_Stage" VARCHAR(20))
returns string
language JavaScript
EXECUTE AS CALLER
as
$$
"option strict"
var sql = `SELECT * FROM ${Trgt_DB}.META.PREPROCESS_META`;
var rs = exec(sql);
var tgtSchemaName;
var tgtTableName;
var retentt;
var retentp;
var remove_data_tbl;
while (rs.next()) {
tgtSchemaName = select_stmt_exec.getColumnValue(5);
tgtTableName = select_stmt_exec.getColumnValue(6);
retentt = select_stmt_exec.getColumnValue(13);
retentp = select_stmt_exec.getColumnValue(12);
remove_data_tbl = `delete from ${tgtSchemaName}.${tgtTableName} A USING META.PREPROCESS_META B
where A.SYS_DATE <= DATEADD('$retentt', -${retentp}, current_date()) AND B.RETENTION_TYPE = '${retentt}' AND B.RETENTION_PERIOD = ${retentp} AND B.TGT_SCHEMA_NAME = '${tgtSchemaName}' AND B.TGT_TABLE_NAME = '${tgtTableName}'`;
exec(remove_data_tbl);
}
// ---- End main function. Start of helper functions.
function exec(sql){
cmd = {sqlText: sql};
stmt = snowflake.createStatement(cmd);
var rs;
rs = stmt.execute();
return rs;
}
$$;
Need help on a small Snowflake stored procedure written in JavaScript, i am getting the following error,
JavaScript compilation error: Uncaught SyntaxError: Unexpected token ':' in ERR_LOGGING at ' , binds: [v_err_seq,APP_NAME,ERR_MSG, MISC_STR,v_time_st])"' position 10
The code is as follows,
create or replace sequence SEQ_ERR;
create or replace table ERROR_LOG(ERR_SEQ NUMBER, APP_NAME VARCHAR(250), ERR_MSG VARCHAR(2500), MISC_STRING VARCHAR(2500), err_date timestamp);
CREATE OR REPLACE PROCEDURE ERR_LOGGING(app_name varchar, err_msg varchar, misc_str varchar )
returns string not null
language javascript
strict volatile
execute as caller
as
$$
try {
var v_time_st = getScalar("SELECT CURRENT_TIMESTAMP");
var v_err_seq = getScalar("SELECT SEQ_ERR.nextval");
error_msg = 'THis is in the line before the execute';
var sqlCmd = `insert into ERROR_LOG (ERR_SEQ , APP_NAME, ERR_MSG , MISC_STRING, ERR_DATE) VALUES (:1, :2, :3, :4, :5)`;
stmt = snowflake.execute({ sqlText: sqlCmd, binds: [v_err_seq, APP_NAME, ERR_MSG, sqlCmd,v_time_st]});
}
catch (err) {
result= " "
result = "Failed: Code: " + err.code + "\n State: " + err.state;
result += "\n Message: " + err.message;
result += "\nStack Trace:\n" + err.stackTraceTxt;
}
function getScalar(queryString) {
var out;
cmd1 = {sqlText: queryString};
stmt = snowflake.createStatement(cmd1);
var rs;
rs = stmt.execute();
rs.next();
return rs.getColumnValue(1);
return out;
}
return result;
$$;
call ERR_LOGGING('TEST_APP', 'THis is an error messge', 'ON line 1');
The problems start on these lines:
var v_time_st = snowflake.execute( {sqlText: "SELECT CURRENT_TIMESTAMP;"} );
var v_err_seq = snowflake.execute( {sqlText: "SELECT SEQ_ERR.nextval;"} );
When you run a SQL statement through the stored procedures API, it returns a table in the form of an object called a ResultSet. You're returning a ResultSet to these variables, not the values you want. In order to get the values, you need to use the .next() method of the ResultSet and then the .getColumnValue() method.
The problem then continues to the next line doing the insert. It's expecting scalar values for the binds but is sent variables containing ResultSets objects with all their attendant complexity.
It's a common practice to need to get single scalar values from a SQL statement. If you know for 100% certain that your query will return exactly one row and one column, you can use a helper function like this one called getScalar. Here's an example:
create or replace procedure FOO()
returns timestamp
language javascript
strict volatile
execute as caller
as
$$
var v_time_st = getScalar("SELECT CURRENT_TIMESTAMP");
return v_time_st;
// Main function above, helper functions below.
function getScalar(queryString) {
var out;
cmd1 = {sqlText: queryString};
stmt = snowflake.createStatement(cmd1);
var rs;
rs = stmt.execute();
rs.next();
return rs.getColumnValue(1);
return out;
}
$$;
call foo();
You can keep that helper function below your main function and out of the way. From there you can do this to get your variables assigned how you need them:
var v_time_st = getScalar("SELECT CURRENT_TIMESTAMP");
var v_err_seq = getScalar("SELECT SEQ_ERR.nextval");
Based on your update, here's refactored code that should work:
CREATE OR REPLACE PROCEDURE ERR_LOGGING(app_name varchar, err_msg varchar, misc_str varchar )
returns string not null
language javascript
strict volatile
execute as caller
as
$$
try {
var v_time_st = getScalar("SELECT CURRENT_TIMESTAMP");
var v_err_seq = getScalar("SELECT SEQ_ERR.nextval");
var sqlCmd = `insert into ERROR_LOG (ERR_SEQ NUMBER, APP_NAME, ERR_MSG , MISC_STRING, ERR_DATE) VALUES (:1, :2, :3, :4, :5)`;
stmt = snowflake.execute({ sqlText: sqlCmd, binds: [P_BATCH_ID, JOB_NAME, error_msg1, sql_command]});
}
catch (err) {
result= " "
result = "Failed: Code: " + err.code + "\n State: " + err.state;
result += "\n Message: " + err.message;
result += "\nStack Trace:\n" + err.stackTraceTxt;
}
function getScalar(queryString) {
var out;
cmd1 = {sqlText: queryString};
stmt = snowflake.createStatement(cmd1);
var rs;
rs = stmt.execute();
rs.next();
return rs.getColumnValue(1);
return out;
}
return result;
$$;
call ERR_LOGGING('TEST_APP', 'THis is an error messge', 'ON line 1');
Can someone help find these zero row tables?
CREATE OR REPLACE PROCEDURE checkrows()
RETURNS VARIANT
LANGUAGE JAVASCRIPT
AS
$
function ExecuteNonQuery(querystring) {
var out = '';
cmd1 = {sqlText: select * from information_schema.tables where rows_count = 0;};
stmt = snowflake.createStatement(cmd1);
var rs;
try{
rs = stmt.execute();
rs.next();
out = "SUCCESS: " + rs.getColumnValue(0);
}
catch(err) {
throw "ERROR: " + err.message.replace(/\n/g, " ");
}enter code here
return out;
}
$$;
If you have a use case that it's okay to get row counts that are several minutes older than the last change (generally 15-90 minutes but up to 3 hours), you can simply run this:
select * from "SNOWFLAKE"."ACCOUNT_USAGE"."TABLES"
where TABLE_TYPE = 'BASE TABLE' and DELETED is null and ROW_COUNT = 0;
Edit: Since this needs to be automated, this SP will return a variant array with objects containing the db, schema, and table name of all tables with zero rows.
create or replace procedure FIND_EMPTY_TABLES(DATABASE_PATTERN string) -- Use .* for all databases in account. It will skip SNOWFLAKE and SNOWFLAKE_SAMPLE_DATA
returns variant
language javascript
execute as owner
as
$$
class Account {constructor(databases){this.databases = databases;}}
class Database {constructor(name) {this.name = name;}}
class Query{constructor(statement){this.statement = statement;}}
var account = getDatabasesInAccount(DATABASE_PATTERN);
var out = [];
for (var i = 0; i < account.databases.length; i++) {
out = out.concat(rsToJSON(getQuery(
`select TABLE_NAME, TABLE_CATALOG, TABLE_SCHEMA, TABLE_OWNER
from ${account.databases[i].name}.INFORMATION_SCHEMA.TABLES
where TABLE_TYPE = 'BASE TABLE' and ROW_COUNT = 0`)));
}
return out;
//------
function getQuery(sql){
cmd1 = {sqlText: sql};
var query = new Query(snowflake.createStatement(cmd1));
query.resultSet = query.statement.execute();
return query;
}
function executeSingleValueQuery(columnName, queryString) {
cmd = {sqlText: queryString};
stmt = snowflake.createStatement(cmd);
var rs;
rs = stmt.execute();
rs.next();
return rs.getColumnValue(columnName);
}
function getDatabasesInAccount(databasePattern){
const SYSTEM_DB_NAMES = ["SNOWFLAKE", "SNOWFLAKE_SAMPLE_DATA"];
var db = executeSingleValueQuery("name", "show databases");
var i = 0;
var dbRS = getResultSet(`select DATABASE_NAME from "${db}".INFORMATION_SCHEMA.DATABASES where rlike (DATABASE_NAME, '${databasePattern}');`);
var databases = [];
var db;
while (dbRS.next()){
db = new Database(dbRS.getColumnValue("DATABASE_NAME"));
if (!SYSTEM_DB_NAMES.includes(db)) {
databases.push(db);
}
}
return new Account(databases);
}
function getResultSet(sql){
let cmd = {sqlText: sql};
let stmt = snowflake.createStatement(cmd);
let rs = stmt.execute();
return rs;
}
function rsToJSON(query) {
var i;
var row = {};
var table = [];
while (query.resultSet.next()) {
for(col = 1; col <= query.statement.getColumnCount(); col++) {
row[query.statement.getColumnName(col)] = query.resultSet.getColumnValue(col);
}
table.push(row);
}
return table;
}
$$;
call FIND_EMPTY_TABLES('.*'); -- .* is the RegExp pattern that tell it to check all databases except built-in ones.
Dirty and quick option adding to gregs excellent answer ... obviously if you like the results add 'create or replace zero_row_count_base_tables as ...' .... set it to re-create the view maybe weekly/daily then run the view when you want.
SELECT
' SELECT TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME from UTIL_DB.INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = \'BASE TABLE\' AND ROW_COUNT > 0 '
UNION
SELECT concat( 'UNION SELECT TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME
FROM ', DATABASES.DATABASE_NAME,'.INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = \'BASE TABLE\' AND ROW_COUNT > 0 ')
FROM UTIL_DB.INFORMATION_SCHEMA.DATABASES
I have a snowflake procedure using javascript.
the procedure exec the statement and return a single value result into a var:
var res = stmt.execute();
I want insert this value into a table. but this value is return as an object.
How do I work around the solution?
My script is dynamic and has a while loop that generates multiple statements, but here is a simplified version to focus on what I need to do get
Thx
create Procedure GetLastUpdateDate()
returns boolean
language javascript
as
$$
sql_command = "select max(Updatedate) from MyTable;";
var stmt = snowflake.createStatement({ sqlText: sql_command});
var res = stmt.execute();
snowflake.createStatement( { sqlText: 'INSERT INTO MYTABLE (NCOLUMN) VALUES (?)',
binds: [res]
} ).execute();
$$
;
.getColumnValue() could be used:
getColumnValue(colIdx|colName)
This method returns the value of a column in the current row (i.e. the row most recently retrieved by next()).
sql_command = "select max(Updatedate) from MyTable;";
var stmt = snowflake.createStatement({ sqlText: sql_command});
var res = stmt.execute();
while (res.next()) {
var myVal = res.getColumnValue(1);
snowflake.createStatement( { sqlText: 'INSERT INTO MYTABLE (NCOLUMN) VALUES (?)',
binds: [myVal]
} ).execute();
}
If more than one row is expected loop should be used: while (res.next()) {...}