I'm trying to find out what protocol the SnowFlake JDBC library uses to communicate with SnowFlake. I see hints here and there that it seems to be using HTTPS as the protocol. Is this true?
To my knowledge, other JDBC libraries like for example for Oracle or PostgreSQL use the lower level TCP protocol to communicate with their database servers, and not the application-level HTTP(S) protocol, so I'm confused.
My organization only supports securely routing http(s)-based communication. Can I use this snowflake jdbc library then?
I have browsed all documentation that I could find, but wasn't able to answer this question.
My issue on GitHub didn't get an answer either.
Edit: Yes, I've seen this question, but I don't feel that it answers my question. SSL/TLS is an encryption, but that doesn't specify the data format.
It looks like the jdbc driver uses HTTP Client HttpUtil.initHttpClient(httpClientSettingsKey, null);, as you can see in here
The HTTP Utility Class is available here
Putting an excerpt of the session open method here in case the link goes bad/dead.
/**
* Open a new database session
*
* #throws SFException this is a runtime exception
* #throws SnowflakeSQLException exception raised from Snowflake components
*/
public synchronized void open() throws SFException, SnowflakeSQLException {
performSanityCheckOnProperties();
Map<SFSessionProperty, Object> connectionPropertiesMap = getConnectionPropertiesMap();
logger.debug(
"input: server={}, account={}, user={}, password={}, role={}, database={}, schema={},"
+ " warehouse={}, validate_default_parameters={}, authenticator={}, ocsp_mode={},"
+ " passcode_in_password={}, passcode={}, private_key={}, disable_socks_proxy={},"
+ " application={}, app_id={}, app_version={}, login_timeout={}, network_timeout={},"
+ " query_timeout={}, tracing={}, private_key_file={}, private_key_file_pwd={}."
+ " session_parameters: client_store_temporary_credential={}",
connectionPropertiesMap.get(SFSessionProperty.SERVER_URL),
connectionPropertiesMap.get(SFSessionProperty.ACCOUNT),
connectionPropertiesMap.get(SFSessionProperty.USER),
!Strings.isNullOrEmpty((String) connectionPropertiesMap.get(SFSessionProperty.PASSWORD))
? "***"
: "(empty)",
connectionPropertiesMap.get(SFSessionProperty.ROLE),
connectionPropertiesMap.get(SFSessionProperty.DATABASE),
connectionPropertiesMap.get(SFSessionProperty.SCHEMA),
connectionPropertiesMap.get(SFSessionProperty.WAREHOUSE),
connectionPropertiesMap.get(SFSessionProperty.VALIDATE_DEFAULT_PARAMETERS),
connectionPropertiesMap.get(SFSessionProperty.AUTHENTICATOR),
getOCSPMode().name(),
connectionPropertiesMap.get(SFSessionProperty.PASSCODE_IN_PASSWORD),
!Strings.isNullOrEmpty((String) connectionPropertiesMap.get(SFSessionProperty.PASSCODE))
? "***"
: "(empty)",
connectionPropertiesMap.get(SFSessionProperty.PRIVATE_KEY) != null
? "(not null)"
: "(null)",
connectionPropertiesMap.get(SFSessionProperty.DISABLE_SOCKS_PROXY),
connectionPropertiesMap.get(SFSessionProperty.APPLICATION),
connectionPropertiesMap.get(SFSessionProperty.APP_ID),
connectionPropertiesMap.get(SFSessionProperty.APP_VERSION),
connectionPropertiesMap.get(SFSessionProperty.LOGIN_TIMEOUT),
connectionPropertiesMap.get(SFSessionProperty.NETWORK_TIMEOUT),
connectionPropertiesMap.get(SFSessionProperty.QUERY_TIMEOUT),
connectionPropertiesMap.get(SFSessionProperty.TRACING),
connectionPropertiesMap.get(SFSessionProperty.PRIVATE_KEY_FILE),
!Strings.isNullOrEmpty(
(String) connectionPropertiesMap.get(SFSessionProperty.PRIVATE_KEY_FILE_PWD))
? "***"
: "(empty)",
sessionParametersMap.get(CLIENT_STORE_TEMPORARY_CREDENTIAL));
HttpClientSettingsKey httpClientSettingsKey = getHttpClientKey();
logger.debug(
"connection proxy parameters: use_proxy={}, proxy_host={}, proxy_port={}, proxy_user={},"
+ " proxy_password={}, non_proxy_hosts={}, proxy_protocol={}",
httpClientSettingsKey.usesProxy(),
httpClientSettingsKey.getProxyHost(),
httpClientSettingsKey.getProxyPort(),
httpClientSettingsKey.getProxyUser(),
!Strings.isNullOrEmpty(httpClientSettingsKey.getProxyPassword()) ? "***" : "(empty)",
httpClientSettingsKey.getNonProxyHosts(),
httpClientSettingsKey.getProxyProtocol());
// TODO: temporarily hardcode sessionParameter debug info. will be changed in the future
SFLoginInput loginInput = new SFLoginInput();
loginInput
.setServerUrl((String) connectionPropertiesMap.get(SFSessionProperty.SERVER_URL))
.setDatabaseName((String) connectionPropertiesMap.get(SFSessionProperty.DATABASE))
.setSchemaName((String) connectionPropertiesMap.get(SFSessionProperty.SCHEMA))
.setWarehouse((String) connectionPropertiesMap.get(SFSessionProperty.WAREHOUSE))
.setRole((String) connectionPropertiesMap.get(SFSessionProperty.ROLE))
.setValidateDefaultParameters(
connectionPropertiesMap.get(SFSessionProperty.VALIDATE_DEFAULT_PARAMETERS))
.setAuthenticator((String) connectionPropertiesMap.get(SFSessionProperty.AUTHENTICATOR))
.setOKTAUserName((String) connectionPropertiesMap.get(SFSessionProperty.OKTA_USERNAME))
.setAccountName((String) connectionPropertiesMap.get(SFSessionProperty.ACCOUNT))
.setLoginTimeout(loginTimeout)
.setAuthTimeout(authTimeout)
.setUserName((String) connectionPropertiesMap.get(SFSessionProperty.USER))
.setPassword((String) connectionPropertiesMap.get(SFSessionProperty.PASSWORD))
.setToken((String) connectionPropertiesMap.get(SFSessionProperty.TOKEN))
.setPasscodeInPassword(passcodeInPassword)
.setPasscode((String) connectionPropertiesMap.get(SFSessionProperty.PASSCODE))
.setConnectionTimeout(httpClientConnectionTimeout)
.setSocketTimeout(httpClientSocketTimeout)
.setAppId((String) connectionPropertiesMap.get(SFSessionProperty.APP_ID))
.setAppVersion((String) connectionPropertiesMap.get(SFSessionProperty.APP_VERSION))
.setSessionParameters(sessionParametersMap)
.setPrivateKey((PrivateKey) connectionPropertiesMap.get(SFSessionProperty.PRIVATE_KEY))
.setPrivateKeyFile((String) connectionPropertiesMap.get(SFSessionProperty.PRIVATE_KEY_FILE))
.setPrivateKeyFilePwd(
(String) connectionPropertiesMap.get(SFSessionProperty.PRIVATE_KEY_FILE_PWD))
.setApplication((String) connectionPropertiesMap.get(SFSessionProperty.APPLICATION))
.setServiceName(getServiceName())
.setOCSPMode(getOCSPMode())
.setHttpClientSettingsKey(httpClientSettingsKey);
// propagate OCSP mode to SFTrustManager. Note OCSP setting is global on JVM.
HttpUtil.initHttpClient(httpClientSettingsKey, null);
SFLoginOutput loginOutput =
SessionUtil.openSession(loginInput, connectionPropertiesMap, tracingLevel.toString());
isClosed = false;
authTimeout = loginInput.getAuthTimeout();
sessionToken = loginOutput.getSessionToken();
masterToken = loginOutput.getMasterToken();
idToken = loginOutput.getIdToken();
mfaToken = loginOutput.getMfaToken();
setDatabaseVersion(loginOutput.getDatabaseVersion());
setDatabaseMajorVersion(loginOutput.getDatabaseMajorVersion());
setDatabaseMinorVersion(loginOutput.getDatabaseMinorVersion());
httpClientSocketTimeout = loginOutput.getHttpClientSocketTimeout();
masterTokenValidityInSeconds = loginOutput.getMasterTokenValidityInSeconds();
setDatabase(loginOutput.getSessionDatabase());
setSchema(loginOutput.getSessionSchema());
setRole(loginOutput.getSessionRole());
setWarehouse(loginOutput.getSessionWarehouse());
setSessionId(loginOutput.getSessionId());
setAutoCommit(loginOutput.getAutoCommit());
// Update common parameter values for this session
SessionUtil.updateSfDriverParamValues(loginOutput.getCommonParams(), this);
String loginDatabaseName = (String) connectionPropertiesMap.get(SFSessionProperty.DATABASE);
String loginSchemaName = (String) connectionPropertiesMap.get(SFSessionProperty.SCHEMA);
String loginRole = (String) connectionPropertiesMap.get(SFSessionProperty.ROLE);
String loginWarehouse = (String) connectionPropertiesMap.get(SFSessionProperty.WAREHOUSE);
if (loginDatabaseName != null && !loginDatabaseName.equalsIgnoreCase(getDatabase())) {
sqlWarnings.add(
new SFException(
ErrorCode.CONNECTION_ESTABLISHED_WITH_DIFFERENT_PROP,
"Database",
loginDatabaseName,
getDatabase()));
}
if (loginSchemaName != null && !loginSchemaName.equalsIgnoreCase(getSchema())) {
sqlWarnings.add(
new SFException(
ErrorCode.CONNECTION_ESTABLISHED_WITH_DIFFERENT_PROP,
"Schema",
loginSchemaName,
getSchema()));
}
if (loginRole != null && !loginRole.equalsIgnoreCase(getRole())) {
sqlWarnings.add(
new SFException(
ErrorCode.CONNECTION_ESTABLISHED_WITH_DIFFERENT_PROP, "Role", loginRole, getRole()));
}
if (loginWarehouse != null && !loginWarehouse.equalsIgnoreCase(getWarehouse())) {
sqlWarnings.add(
new SFException(
ErrorCode.CONNECTION_ESTABLISHED_WITH_DIFFERENT_PROP,
"Warehouse",
loginWarehouse,
getWarehouse()));
}
// start heartbeat for this session so that the master token will not expire
startHeartbeatForThisSession();
}
I'm going to create a SQL server database in RDS using Terraform. My Terraform file looks like this:
### RDS ###
# Subnet Group
resource "aws_db_subnet_group" "private" {
name = "db_arcgis-${var.env_name}-dbsubnet"
description = "Subnet Group for Arcgis ${var.env_tag}} DB"
subnet_ids = ["${aws_subnet.public1.id}", "${aws_subnet.public2.id}"]
tags {
Env = "${var.env_tag}"
}
}
# RDS DB parameter group
# Must enabled triggers to allow Multi-AZ
resource "aws_db_parameter_group" "allow_triggers" {
name = "arcgis-${var.env_name}-allow-triggers"
family = "sqlserver-se-12.0"
description = "Parameter Group for Arcgis ${var.env_tag} to allow triggers"
parameter {
name = "log_bin_trust_function_creators"
value = "1"
}
tags {
Env = "${var.env_tag}"
}
}
# RDS
resource "aws_db_instance" "main" {
allocated_storage = "${var.db_size}"
engine = "${var.db_engine}"
engine_version = "${var.db_version}"
instance_class = "${var.db_instance}"
identifier = "arcgis-${var.env_name}-db"
name = "${var.db_name}"
username = "${var.db_username}"
password = "${var.db_password}"
db_subnet_group_name = "${aws_db_subnet_group.private.id}"
parameter_group_name = "${aws_db_parameter_group.allow_triggers.id}"
multi_az = "${var.db_multiaz}"
vpc_security_group_ids = ["${aws_security_group.private_rds.id}"]
#availability_zone = "${var.vpc_az1}"
publicly_accessible = "true"
backup_retention_period = "2"
apply_immediately = "true"
tags {
Env = "${var.env_tag}"
}
}
I get this error by applying the Terraform files:
Error applying plan:
1 error(s) occurred:
* aws_db_parameter_group.allow_triggers: Error modifying DB Parameter Group: InvalidParameterValue: Could not find parameter with name: log_bin_trust_function_creators
status code: 400, request id: d298ab14-8b94-11e6-a088-31e21873c378
The obvious issue here is that log_bin_trust_function_creators isn't an available parameter for the sqlserver-se-12.0 parameter group family as you can see here when listing all the parameters in a parameter group based on sqlserver-se-12.0:
$ aws rds describe-db-parameters --db-parameter-group-name test-sqlserver-se-12-0 --query 'Parameters[*].ParameterName'
[
"1204",
"1211",
"1222",
"1224",
"2528",
"3205",
"3226",
"3625",
"4199",
"4616",
"6527",
"7806",
"access check cache bucket count",
"access check cache quota",
"ad hoc distributed queries",
"affinity i/o mask",
"affinity mask",
"agent xps",
"allow updates",
"backup compression default",
"blocked process threshold (s)",
"c2 audit mode",
"clr enabled",
"contained database authentication",
"cost threshold for parallelism",
"cross db ownership chaining",
"cursor threshold",
"database mail xps",
"default full-text language",
"default language",
"default trace enabled",
"disallow results from triggers",
"filestream access level",
"fill factor (%)",
"ft crawl bandwidth (max)",
"ft crawl bandwidth (min)",
"ft notify bandwidth (max)",
"ft notify bandwidth (min)",
"in-doubt xact resolution",
"index create memory (kb)",
"lightweight pooling",
"locks",
"max degree of parallelism",
"max full-text crawl range",
"max server memory (mb)",
"max text repl size (b)",
"max worker threads",
"media retention",
"min memory per query (kb)",
"min server memory (mb)",
"nested triggers",
"network packet size (b)",
"ole automation procedures",
"open objects",
"optimize for ad hoc workloads",
"ph timeout (s)",
"priority boost",
"query governor cost limit",
"query wait (s)",
"recovery interval (min)",
"remote access",
"remote admin connections",
"remote login timeout (s)",
"remote proc trans",
"remote query timeout (s)",
"replication xps",
"scan for startup procs",
"server trigger recursion",
"set working set size",
"show advanced options",
"smo and dmo xps",
"transform noise words",
"two digit year cutoff",
"user connections",
"user options",
"xp_cmdshell"
]
Instead that parameter is only available in MySQL flavours:
$ aws rds describe-db-parameters --db-parameter-group-name default.mysql5.6 --query 'Parameters[*].ParameterName'
[
"allow-suspicious-udfs",
"auto_increment_increment",
"auto_increment_offset",
"autocommit",
"automatic_sp_privileges",
"back_log",
"basedir",
"binlog_cache_size",
"binlog_checksum",
"binlog_error_action",
"binlog_format",
"binlog_max_flush_queue_time",
"binlog_order_commits",
"binlog_row_image",
"binlog_rows_query_log_events",
"binlog_stmt_cache_size",
"binlogging_impossible_mode",
"bulk_insert_buffer_size",
"character-set-client-handshake",
"character_set_client",
"character_set_connection",
"character_set_database",
"character_set_filesystem",
"character_set_results",
"character_set_server",
"collation_connection",
"collation_server",
"completion_type",
"concurrent_insert",
"connect_timeout",
"core-file",
"datadir",
"default_storage_engine",
"default_time_zone",
"default_tmp_storage_engine",
"default_week_format",
"delay_key_write",
"delayed_insert_limit",
"delayed_insert_timeout",
"delayed_queue_size",
"div_precision_increment",
"end_markers_in_json",
"enforce_gtid_consistency",
"eq_range_index_dive_limit",
"event_scheduler",
"explicit_defaults_for_timestamp",
"flush",
"flush_time",
"ft_boolean_syntax",
"ft_max_word_len",
"ft_min_word_len",
"ft_query_expansion_limit",
"ft_stopword_file",
"general_log",
"general_log_file",
"group_concat_max_len",
"gtid-mode",
"host_cache_size",
"init_connect",
"innodb_adaptive_flushing",
"innodb_adaptive_flushing_lwm",
"innodb_adaptive_hash_index",
"innodb_adaptive_max_sleep_delay",
"innodb_autoextend_increment",
"innodb_autoinc_lock_mode",
"innodb_buffer_pool_dump_at_shutdown",
"innodb_buffer_pool_dump_now",
"innodb_buffer_pool_filename",
"innodb_buffer_pool_instances",
"innodb_buffer_pool_load_abort",
"innodb_buffer_pool_load_at_startup",
"innodb_buffer_pool_load_now",
"innodb_buffer_pool_size",
"innodb_change_buffer_max_size",
"innodb_change_buffering",
"innodb_checksum_algorithm",
"innodb_cmp_per_index_enabled",
"innodb_commit_concurrency",
"innodb_compression_failure_threshold_pct",
"innodb_compression_level",
"innodb_compression_pad_pct_max",
"innodb_concurrency_tickets",
"innodb_data_home_dir",
"innodb_fast_shutdown",
"innodb_file_format",
"innodb_file_per_table",
"innodb_flush_log_at_timeout",
"innodb_flush_log_at_trx_commit",
"innodb_flush_method",
"innodb_flush_neighbors",
"innodb_flushing_avg_loops",
"innodb_force_load_corrupted",
"innodb_ft_aux_table",
"innodb_ft_cache_size",
"innodb_ft_enable_stopword",
"innodb_ft_max_token_size",
"innodb_ft_min_token_size",
"innodb_ft_num_word_optimize",
"innodb_ft_result_cache_limit",
"innodb_ft_server_stopword_table",
"innodb_ft_sort_pll_degree",
"innodb_ft_user_stopword_table",
"innodb_io_capacity",
"innodb_io_capacity_max",
"innodb_large_prefix",
"innodb_lock_wait_timeout",
"innodb_log_buffer_size",
"innodb_log_compressed_pages",
"innodb_log_file_size",
"innodb_log_group_home_dir",
"innodb_lru_scan_depth",
"innodb_max_dirty_pages_pct",
"innodb_max_purge_lag",
"innodb_max_purge_lag_delay",
"innodb_monitor_disable",
"innodb_monitor_enable",
"innodb_monitor_reset",
"innodb_monitor_reset_all",
"innodb_old_blocks_pct",
"innodb_old_blocks_time",
"innodb_online_alter_log_max_size",
"innodb_open_files",
"innodb_optimize_fulltext_only",
"innodb_page_size",
"innodb_print_all_deadlocks",
"innodb_purge_batch_size",
"innodb_purge_threads",
"innodb_random_read_ahead",
"innodb_read_ahead_threshold",
"innodb_read_io_threads",
"innodb_read_only",
"innodb_replication_delay",
"innodb_rollback_on_timeout",
"innodb_rollback_segments",
"innodb_sort_buffer_size",
"innodb_spin_wait_delay",
"innodb_stats_auto_recalc",
"innodb_stats_method",
"innodb_stats_on_metadata",
"innodb_stats_persistent",
"innodb_stats_persistent_sample_pages",
"innodb_stats_transient_sample_pages",
"innodb_strict_mode",
"innodb_support_xa",
"innodb_sync_array_size",
"innodb_sync_spin_loops",
"innodb_table_locks",
"innodb_thread_concurrency",
"innodb_thread_sleep_delay",
"innodb_undo_directory",
"innodb_undo_logs",
"innodb_undo_tablespaces",
"innodb_use_native_aio",
"innodb_write_io_threads",
"interactive_timeout",
"join_buffer_size",
"keep_files_on_create",
"key_buffer_size",
"key_cache_age_threshold",
"key_cache_block_size",
"key_cache_division_limit",
"lc_time_names",
"local_infile",
"lock_wait_timeout",
"log-bin",
"log_bin_trust_function_creators",
"log_bin_use_v1_row_events",
"log_error",
"log_output",
"log_queries_not_using_indexes",
"log_slave_updates",
"log_slow_admin_statements",
"log_slow_slave_statements",
"log_throttle_queries_not_using_indexes",
"log_warnings",
"long_query_time",
"low_priority_updates",
"lower_case_table_names",
"master-info-repository",
"master_verify_checksum",
"max_allowed_packet",
"max_binlog_cache_size",
"max_binlog_size",
"max_binlog_stmt_cache_size",
"max_connect_errors",
"max_connections",
"max_delayed_threads",
"max_error_count",
"max_heap_table_size",
"max_insert_delayed_threads",
"max_join_size",
"max_length_for_sort_data",
"max_prepared_stmt_count",
"max_seeks_for_key",
"max_sort_length",
"max_sp_recursion_depth",
"max_tmp_tables",
"max_user_connections",
"max_write_lock_count",
"metadata_locks_cache_size",
"min_examined_row_limit",
"myisam_data_pointer_size",
"myisam_max_sort_file_size",
"myisam_mmap_size",
"myisam_sort_buffer_size",
"myisam_stats_method",
"myisam_use_mmap",
"net_buffer_length",
"net_read_timeout",
"net_retry_count",
"net_write_timeout",
"old-style-user-limits",
"old_passwords",
"optimizer_prune_level",
"optimizer_search_depth",
"optimizer_switch",
"optimizer_trace",
"optimizer_trace_features",
"optimizer_trace_limit",
"optimizer_trace_max_mem_size",
"optimizer_trace_offset",
"performance_schema",
"performance_schema_accounts_size",
"performance_schema_digests_size",
"performance_schema_events_stages_history_long_size",
"performance_schema_events_stages_history_size",
"performance_schema_events_statements_history_long_size",
"performance_schema_events_statements_history_size",
"performance_schema_events_waits_history_long_size",
"performance_schema_events_waits_history_size",
"performance_schema_hosts_size",
"performance_schema_max_cond_classes",
"performance_schema_max_cond_instances",
"performance_schema_max_file_classes",
"performance_schema_max_file_handles",
"performance_schema_max_file_instances",
"performance_schema_max_mutex_classes",
"performance_schema_max_mutex_instances",
"performance_schema_max_rwlock_classes",
"performance_schema_max_rwlock_instances",
"performance_schema_max_socket_classes",
"performance_schema_max_socket_instances",
"performance_schema_max_stage_classes",
"performance_schema_max_statement_classes",
"performance_schema_max_table_handles",
"performance_schema_max_table_instances",
"performance_schema_max_thread_classes",
"performance_schema_max_thread_instances",
"performance_schema_session_connect_attrs_size",
"performance_schema_setup_actors_size",
"performance_schema_setup_objects_size",
"performance_schema_users_size",
"pid_file",
"plugin_dir",
"port",
"preload_buffer_size",
"profiling_history_size",
"query_alloc_block_size",
"query_cache_limit",
"query_cache_min_res_unit",
"query_cache_size",
"query_cache_type",
"query_cache_wlock_invalidate",
"query_prealloc_size",
"range_alloc_block_size",
"read_buffer_size",
"read_only",
"read_rnd_buffer_size",
"relay-log",
"relay_log_info_repository",
"relay_log_recovery",
"safe-user-create",
"secure_auth",
"secure_file_priv",
"server_id",
"simplified_binlog_gtid_recovery",
"skip-character-set-client-handshake",
"skip-slave-start",
"skip_external_locking",
"skip_name_resolve",
"skip_show_database",
"slave_checkpoint_group",
"slave_checkpoint_period",
"slave_parallel_workers",
"slave_pending_jobs_size_max",
"slave_sql_verify_checksum",
"slave_type_conversions",
"slow_launch_time",
"slow_query_log",
"slow_query_log_file",
"socket",
"sort_buffer_size",
"sql_mode",
"sql_select_limit",
"stored_program_cache",
"sync_binlog",
"sync_frm",
"sync_master_info",
"sync_relay_log",
"sync_relay_log_info",
"sysdate-is-now",
"table_definition_cache",
"table_open_cache",
"table_open_cache_instances",
"temp-pool",
"thread_cache_size",
"thread_stack",
"time_zone",
"timed_mutexes",
"tmp_table_size",
"tmpdir",
"transaction_alloc_block_size",
"transaction_prealloc_size",
"tx_isolation",
"updatable_views_with_limit",
"validate-password",
"validate_password_dictionary_file",
"validate_password_length",
"validate_password_mixed_case_count",
"validate_password_number_count",
"validate_password_policy",
"validate_password_special_char_count",
"wait_timeout"
]