Skip to main content
Version: 0.0.1

Source Configuration

Dremio supports a variety of source types, and the available parameters of the config object differ among the source types. This page provides examples of the config object for each supported source type and descriptions of the config object's parameters that are available for each source type.

note:

The source object contains the same parameters for all source types except for the parameters of the config object. Read the Source page for information about the parameters all source types have in common.

Amazon Redshift​

{{< codeheader "Amazon Redshift Source config Object" >}}

{
  "config": {
    "connectionString": "jdbc:redshift://testing-redshift-cluster.r57mkjay4utq.us-west-2.redshift.amazonaws.com:5439/support",
    "username": "redshift-owner",
    "password": "$DREMIO_EXISTING_VALUE$",
    "authenticationType": "MASTER",
    "propertyList": [
      {
        "name": "ApplicationName",
        "value": "testingDashboard"
      }
    ],
    "fetchSize": 200,
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "queryTimeoutSec": 0
  }
}

Amazon Redshift config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="amazonRedshiftParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="amazonRedshiftpropertyListParameters" >}}

Amazon S3​

{{< codeheader "Amazon S3 Source config Object" >}}

{
  "config": {
    "accessKey": "EXAMPLE78HT89VS4YJEL",
    "accessSecret": "$DREMIO_EXISTING_VALUE$",
    "secure": true,
    "externalBucketList": [
      "external_bucket_1",
      "external_bucket_2"
    ],
    "propertyList": [
      {
        "name": "fs.s3a.proxy.host",
        "value": "proxyHost.example.com"
      }
    ],
    "rootPath": "/",
    "enableAsync": true,
    "compatibilityMode": false,
    "isCachingEnabled": true,
    "maxCacheSpacePct": 100,
    "whitelistedBuckets": [
      "archive.dremio.com",
      "logs_east-1",
      "logs_west-1"
    ],
    "requesterPays": false,
    "enableFileStatusCheck": true,
    "defaultCtasFormat": "ICEBERG",
    "isPartitionInferenceEnabled": false,
    "credentialType": "ACCESS_KEY"
  }
}

Amazon S3 config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="amazonS3Parameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="amazonS3propertyListParameters" >}}

AWS Glue Data Catalog​

{{< codeheader "AWS Glue Data Catalog Source config Object" >}}

{
  "config": {
    "regionNameSelection": "US_EAST_1",
    "accessKey": "EXAMPLE78HT89VS4YJEL",
    "accessSecret": "$DREMIO_EXISTING_VALUE$",
    "secure": true,
    "enableAsync": true,
    "isCachingEnabled": true,
    "maxCacheSpacePct": 100,
    "assumedRoleARN": "arn:aws:iam::594632595346:role/OrganizationAccountAccessRole",
    "lakeFormationEnableAccessPermissions": false,
    "defaultCtasFormat": "ICEBERG",
    "propertyList": [
      {
        "name": "hive.metastore.warehouse.dir",
        "value": "s3a://example.com/buckets/glue_location"
      }
    ],
    "credentialType": "ACCESS_KEY"
  }
}

Amazon Glue Data Catalog config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="awsGlueDataCatalogParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="awsGlueDataCatalogpropertyListParameters" >}}

Azure Data Lake Storage Gen1​

{{< codeheader "Azure Data Lake Storage Gen1 Source config Object" >}}

{
  "config": {
    "accountName": "datalake1",
    "clientId": "98c23728-3971-4de3-a15f-61f483f51eca",
    "clientKeyRefreshUrl": "https://login.microsoftonline.com/88ba5c34-c9af-11ed-b49f-325096b39f47/oauth2/token",
    "clientKeyPassword": "$DREMIO_EXISTING_VALUE$",
    "rootPath": "/",
    "enableAsync": true,
    "isCachingEnabled": true,
    "maxCacheSpacePct": 100,
    "defaultCtasFormat": "ICEBERG",
    "propertyList": [
      {
        "name": "dfs.adls.oauth2.access.token.provider.type",
        "value": "ClientCredential"
      }
    ],
    "isPartitionInferenceEnabled": false
  }
}

Azure Data Lake Storage Gen1 config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="azureDataLakeStorageGen1Parameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="azureDataLakeStorageGen1propertyListParameters" >}}

Azure Storage​

{{< codeheader "Azure Storage Source config Object" >}}

{
  "config": {
    "accountKind": "STORAGE_V2",
    "accountName": "azurestoragev2accountname",
    "accessKey": "$DREMIO_EXISTING_VALUE$",
    "rootPath": "/",
    "enableSSL": true,
    "enableAsync": true,
    "credentialsType": "ACCESS_KEY",
    "isCachingEnabled": true,
    "maxCacheSpacePct": 100,
    "defaultCtasFormat": "ICEBERG",
    "propertyList": [
      {
        "name": "dremio.azure.mode",
        "value": "STORAGE_V2"
      }
    ],
    "isPartitionInferenceEnabled": false
  }
}

Azure Storage config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="azureStorageParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="azureStoragepropertyListParameters" >}}

Dremio-to-Dremio Connector​

{{< codeheader "Dremio-to-Dremio Connector Source config Object" >}}

{
  "config": {
    "hostType": "DIRECT",
    "hostname": "35.219.202.153",
    "port": "31010",
    "username": "exampleuser",
    "password": "$DREMIO_EXISTING_VALUE$",
    "useSsl": false,
    "userImpersonation": false,
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "queryTimeoutSec": 0,
    "propertyList": [
      {
        "name": "disableCertificateVerification",
        "value": "false"
      }
    ]
  }
}

Dremio-to-Dremio Connector config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="dremioToDremioConnectorParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="dremioToDremioConnectorpropertyListParameters" >}}

Elasticsearch​

{{< codeheader "Elasticsearch Source config Object" >}}

{
  "config": {
    "scriptsEnabled": true,
    "showHiddenIndices": false,
    "showIdColumn": false,
    "readTimeoutMillis": 60000,
    "scrollTimeoutMillis": 300000,
    "usePainless": true,
    "scrollSize": 4000,
    "allowPushdownOnNormalizedOrAnalyzedFields": false,
    "warnOnRowCountMismatch": false,
    "encryptionValidationMode": "CERTIFICATE_AND_HOSTNAME_VALIDATION",
    "forceDoublePrecision": false,
    "hostList": [
      {
        "hostname": "172.25.0.208",
        "port": 9200
      }
    ],
    "authenticationType": "ANONYMOUS",
    "sslEnabled": false,
    "useWhitelist": false
  }
}

Elasticsearch config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="elasticsearchParameters" >}}

hostList​

{{< api-section file="data/software-api/source-types-config.json" data="elasticsearchhostListParameters" >}}

Google Cloud Storage​

{{< codeheader "Google Cloud Storage Source config Object" >}}

{
  "config": {
    "projectId": "dremio-4321",
    "authMode": "AUTO",
    "rootPath": "/",
    "asyncEnabled": true,
    "cachingEnable": true,
    "cachePercent": 70,
    "privateKeyId": "",
    "clientEmail": "",
    "clientId": "",
    "defaultCtasFormat": "ICEBERG",
    "propertyList": [
      {
        "name": "dremio.gcs.clientEmail",
        "value": "exampleuser@dremio-project.iam.gserviceaccount.com"
      }
    ],
    "isPartitionInferenceEnabled": false
  }
}

Google Cloud Storage config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="googleCloudStorageParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="googleCloudStoragepropertyListParameters" >}}

Hadoop Distributed File System (HDFS)​

{{< codeheader "HDFS Source config Object" >}}

{
  "config": {
    "hostname": "172.23.0.208",
    "port": 8020,
    "enableImpersonation": false,
    "rootPath": "/",
    "shortCircuitFlag": "SYSTEM",
    "enableAsync": true,
    "isCachingEnabled": false,
    "maxCacheSpacePct": 100,
    "defaultCtasFormat": "ICEBERG",
    "isPartitionInferenceEnabled": false,
    "impersonationUserDelegationMode": "AS_IS",
    "propertyList": [
      {
        "name": "dfs.client.socket-timeout",
        "value": "120000"
      }
    ],
    "vdsAccessDelegationEnabled": true
  }
}

HDFS config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="hdfsParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="hdfspropertyListParameters" >}}

Hive 2.x​

{{< codeheader "Hive 2.x Source config Object" >}}

{
  "config": {
    "hostname": "172.23.0.208",
    "port": 9083,
    "enableSasl": false,
    "propertyList": [
      {
        "name": "hive.server2.enable.doAs",
        "value": "false"
      }
    ],
    "enableAsync": true,
    "isCachingEnabledForS3AndAzureStorage": true,
    "isCachingEnabledForHDFS": true,
    "maxCacheSpacePct": 100,
    "defaultCtasFormat": "ICEBERG",
    "authType": "STORAGE",
    "rangerServiceName": "",
    "rangerHostURL": "",
    "impersonationUserDelegationMode": "AS_IS",
    "vdsAccessDelegationEnabled": true
  }
}

Hive 2.x config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="hive2Parameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="hive2propertyListParameters" >}}

Hive 3.x​

{{< codeheader "Hive 3.x Source config Object" >}}

{
  "config": {
    "hostname": "172.23.0.177",
    "port": 9084,
    "enableSasl": false,
    "propertyList": [
      {
        "name": "fs.s3a.aws.credentials.provider",
        "value": "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"
      }
    ],
    "enableAsync": true,
    "isCachingEnabledForS3AndAzureStorage": true,
    "isCachingEnabledForHDFS": false,
    "maxCacheSpacePct": 100,
    "defaultCtasFormat": "ICEBERG",
    "authType": "STORAGE",
    "rangerServiceName": "",
    "rangerHostURL": "",
    "impersonationUserDelegationMode": "AS_IS",
    "vdsAccessDelegationEnabled": true
  }
}

Hive 3.x config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="hive3Parameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="hive3propertyListParameters" >}}

IBM Db2​

{{< codeheader "IBM Db2 Source config Object" >}}

{
  "config": {
    "database": "tpch",
    "hostname": "172.25.1.245",
    "username": "exampleuser",
    "password": "$DREMIO_EXISTING_VALUE$",
    "port": "50000",
    "fetchSize": 200,
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "queryTimeoutSec": 0,
    "propertyList": [
      {
        "name": "deferPrepares",
        "value": "true"
      }
    ]
  }
}

IBM Db2 config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="ibmDb2Parameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="ibmDb2propertyListParameters" >}}

MapR File System (MapR FS)​

{{< codeheader "MapR FS Source config Object" >}}

{
  "config": {
    "clusterName": "my.cluster.com",
    "enableImpersonation": false,
    "secure": false,
    "rootPath": "/",
    "defaultCtasFormat": "ICEBERG",
    "isPartitionInferenceEnabled": false,
    "impersonationUserDelegationMode": "AS_IS",
    "vdsAccessDelegationEnabled": true
  }
}

MapR FS config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="maprFSParameters" >}}

Microsoft Azure Data Explorer​

{{< codeheader "Microsoft Azure Data Explorer Source config Object" >}}

{
  "config": {
    "clusterUri": "https://prod.eastus.kusto.windows.net",
    "tenantId": "6e334262-b0c6-4c36-9faf-380e97e42964",
    "appId": "c3e4554f-514a-4912-8b75-93800f0d6c71",
    "appSecret": "$DREMIO_EXISTING_VALUE$",
    "databaseName": "datatypes",
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "queryTimeoutSec": 0
  }
}

Microsoft Azure Date Explorer config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="microsoftAzureDataExplorerParameters" >}}

Microsoft Azure Synapse Analytics​

{{< codeheader "Microsoft Azure Synapse Analytics Source config Object" >}}

{
  "config": {
    "hostname": "synapsedata.sql.azuresynapse.net",
    "port": "1433",
    "username": "exampleuser",
    "password": "$DREMIO_EXISTING_VALUE$",
    "authenticationType": "MASTER",
    "fetchSize": 200,
    "useSsl": false,
    "enableServerVerification": true,
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "queryTimeoutSec": 0,
    "database": "dedicatedpool",
    "propertyList": [
      {
        "name": "applicationIntent",
        "value": "ReadWrite"
      }
    ]
  }
}

Microsoft Azure Synapse Analytics config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="microsoftAzureSynapseAnalyticsParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="microsoftAzureSynapseAnalyticspropertyListParameters" >}}

Microsoft SQL Server​

{{< codeheader "Microsoft SQL Server Source config Object" >}}

{
  "config": {
    "hostname": "172.25.0.10",
    "port": "1433",
    "username": "tpch",
    "password": "$DREMIO_EXISTING_VALUE$",
    "authenticationType": "MASTER",
    "fetchSize": 200,
    "useSsl": false,
    "enableServerVerification": true,
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "queryTimeoutSec": 0,
    "showOnlyConnectionDatabase": false,
    "propertyList": [
      {
        "name": "applicationIntent",
        "value": "ReadWrite"
      }
    ]
  }
}

Microsoft SQL Server config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="microsoftSQLServerParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="microsoftSQLServerpropertyListParameters" >}}

MongoDB​

{{< codeheader "MongoDB Source config Object" >}}

{
  "config": {
    "hostList": [
      {
        "hostname": "172.23.0.208",
        "port": 27017
      }
    ],
    "useSsl": false,
    "authenticationType": "ANONYMOUS",
    "authDatabase": "admin",
    "authenticationTimeoutMillis": 2000,
    "secondaryReadsOnly": false,
    "subpartitionSize": 0,
    "sampleSize": 4095,
    "sampleMethod": "FIRST",
    "propertyList": [
      {
        "name": "maxPoolSize",
        "value": "100"
      }
    ],
    "useCaseInsensitiveFieldNames": false
  }
}

MongoDB config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="mongoDBParameters" >}}

hostList​

{{< api-section file="data/software-api/source-types-config.json" data="mongoDBhostListParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="mongoDBpropertyListParameters" >}}

MySQL​

{{< codeheader "MySQL Source config Object" >}}

{
  "config": {
    "hostname": "rdbms-linux-20231031.c.company-4321.internal",
    "port": "3306",
    "username": "exampleuser",
    "password": "$DREMIO_EXISTING_VALUE$",
    "authenticationType": "MASTER",
    "fetchSize": 200,
    "netWriteTimeout": 60,
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "propertyList": [
      {
        "name": "DataSource",
        "value": "sales2021"
      }
    ],
    "queryTimeoutSec": 0
  }
}

mySQL config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="mySQLParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="mySQLpropertyListParameters" >}}

Network-attached Storage (NAS)​

{{< codeheader "NAS Source config Object" >}}

{
  "config": {
    "path": "/home/dremio/",
    "defaultCtasFormat": "ICEBERG",
    "isPartitionInferenceEnabled": false
  }
}

NAS config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="nasParameters" >}}

Oracle​

{{< codeheader "Oracle Source config Object" >}}

{
  "config": {
    "hostname": "172.25.0.10",
    "port": "1521",
    "instance": "xe",
    "username": "exampleuser",
    "password": "$DREMIO_EXISTING_VALUE$",
    "authenticationType": "MASTER",
    "fetchSize": 200,
    "useSsl": false,
    "useTimezoneAsRegion": true,
    "includeSynonyms": false,
    "useLdap": false,
    "nativeEncryption": "ACCEPTED",
    "useKerberos": false,
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "mapDateToTimestamp": true,
    "queryTimeoutSec": 0,
    "propertyList": [
      {
        "name": "connect_timeout",
        "value": "0"
      }
    ]
  }
}

Oracle config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="oracleParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="oraclepropertyListParameters" >}}

PostgreSQL​

{{< codeheader "PostgreSQL Source config Object" >}}

{
  "config": {
    "hostname": "172.25.0.10",
    "port": "5432",
    "databaseName": "postgres",
    "username": "exampleuser",
    "password": "$DREMIO_EXISTING_VALUE$",
    "authenticationType": "MASTER",
    "fetchSize": 200,
    "useSsl": false,
    "encryptionValidationMode": "CERTIFICATE_AND_HOSTNAME_VALIDATION",
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "queryTimeoutSec": 0,
    "propertyList": [
      {
        "name": "connect_timeout",
        "value": "10"
      }
    ]
  }
}

PostgreSQLconfig Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="postgreSQLParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="postgreSQLpropertyListParameters" >}}

Snowflake​

{{< codeheader "Snowflake Source config Object" >}}

{
  "config": {
    "schema": "myschema",
    "hostname": "https://cxazbmq-pa44061.snowflakecomputing.com",
    "database": "SNOWFLAKE_SAMPLE_DATA",
    "role": "ACCOUNTADMIN",
    "warehouse": "COMPUTE_WH",
    "username": "exampleuser1",
    "password": "$DREMIO_EXISTING_VALUE$",
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "queryTimeoutSec": 0,
    "propertyList": [
      {
        "name": "loginTimeout",
        "value": "60"
      }
    ]
  }
}

Snowflake config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="snowflakeParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="snowflakepropertyListParameters" >}}

Teradata​

{{< codeheader "Teradata Source config Object" >}}

{
  "config": {
    "hostname": "teradata.drem.io",
    "port": "1025",
    "database": "dremio_source",
    "username": "dremio",
    "password": "$DREMIO_EXISTING_VALUE$",
    "fetchSize": 200,
    "showOnlyConnectionDatabase": false,
    "useSsl": false,
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "queryTimeoutSec": 0,
    "propertyList": [
      {
        "name": "maxPoolSize",
        "value": "10"
      }
    ]
  }
}

Teradata config Parameters​

{{< api-section file="data/software-api/source-types-config.json" data="teradataParameters" >}}

propertyList​

{{< api-section file="data/software-api/source-types-config.json" data="teradatapropertyListParameters" >}}