Skip to main content
Version: current

Source

Use the Catalog API to retrieve information about sources and the child objects they contain, as well as to create, update, and delete sources.

note:

Dremio supports a number of different source types. Each source type has the same parameters except for the parameters within the config object. The available parameters in the config object are different for each source type. The examples on this page use an Amazon S3 source to demonstrate the available requests and responses for sources. Read Source Configuration for information about the available parameters in the config object for each supported source type.

Source Object
{
"entityType": "source",
"config": {
"accessKey": "EXAMPLE78HT89VS4YJEL",
"accessSecret": "$DREMIO_EXISTING_VALUE$",
"secure": true,
"rootPath": "/",
"enableAsync": true,
"compatibilityMode": false,
"isCachingEnabled": true,
"maxCacheSpacePct": 100,
"whitelistedBuckets": [
"archive.dremio.com",
"logs_east-1",
"logs_west-1"
],
"requesterPays": false,
"enableFileStatusCheck": true,
"defaultCtasFormat": "ICEBERG",
"isPartitionInferenceEnabled": false,
"credentialType": "ACCESS_KEY"
},
"id": "2b1be882-7012-4a99-8d6c-82e32e4562e4",
"tag": "T0/Zr1FOY3A=",
"type": "S3",
"name": "AWS-S3_testgroup",
"createdAt": "2023-02-17T14:32:20.640Z",
"metadataPolicy": {
"authTTLMs": 86400000,
"namesRefreshMs": 3600000,
"datasetRefreshAfterMs": 3600000,
"datasetExpireAfterMs": 10800000,
"datasetUpdateMode": "PREFETCH_QUERIED",
"deleteUnavailableDatasets": true,
"autoPromoteDatasets": false
},
"accelerationGracePeriodMs": 10800000,
"accelerationRefreshPeriodMs": 3600000,
"accelerationNeverExpire": false,
"accelerationNeverRefresh": false,
"children": [
{
"id": "dremio:/AWS-S3_testgroup/archive.dremio.com",
"path": [
"AWS-S3_testgroup",
"archive.dremio.com"
],
"type": "CONTAINER",
"containerType": "FOLDER"
},
{
"id": "dremio:/AWS-S3_testgroup/logs_east-1",
"path": [
"AWS-S3_testgroup",
"logs_east-1"
],
"type": "CONTAINER",
"containerType": "FOLDER"
},
{
"id": "dremio:/AWS-S3_testgroup/logs_west-1",
"path": [
"AWS-S3_testgroup",
"logs_west-1"
],
"type": "CONTAINER",
"containerType": "FOLDER"
}
],
"allowCrossSourceSelection": false,
"disableMetadataValidityCheck": false,
"accessControlList": {
"users": [
{
"id": "ebe519ab-20e3-43ff-9b4c-b3ec590c7e65",
"permissions": [
"VIEW_REFLECTION",
"SELECT"
]
}
],
"roles": [
{
"id": "c45ff4d8-e910-4f85-89db-9b8c29188a56",
"permissions": [
"ALTER",
"CREATE_TABLE",
"DROP",
"INSERT",
"DELETE",
"UPDATE",
"TRUNCATE",
"VIEW_REFLECTION",
"ALTER_REFLECTION",
"MODIFY",
"MANAGE_GRANTS",
"SELECT"
]
}
]
},
"permissions": [
"READ",
"WRITE",
"ALTER_REFLECTION",
"SELECT",
"ALTER",
"VIEW_REFLECTION",
"MODIFY",
"MANAGE_GRANTS",
"CREATE_TABLE",
"DROP",
"EXTERNAL_QUERY",
"INSERT",
"TRUNCATE",
"DELETE",
"UPDATE",
"EXECUTE",
"CREATE_SOURCE",
"ALL"
],
"checkTableAuthorizer": true,
"owner": {
"ownerId": "4fb93af3-acc2-4b10-ad4b-64dd7070d365",
"ownerType": "USER"
}
}

Source Attributes

{{< api-section file="data/software-api/source.json" data="sourceAttributes" >}}

config

{{< api-section file="data/software-api/source.json" data="configAttributes" >}}

The config object attributes vary for different source types. Read Source Configuration for information about the available parameters in the config object for each supported source type.

metadataPolicy

{{< api-section file="data/software-api/source.json" data="metadataPolicyAttributes" >}}

children

{{< api-section file="data/software-api/source.json" data="childrenAttributes" >}}

accessControlList

{{< api-section file="data/software-api/source.json" data="accessControlListAttributes" >}}

owner

{{< api-section file="data/software-api/source.json" data="ownerAttributes" >}}

users and roles

{{< api-section file="data/software-api/source.json" data="usersRolesAccessControlAttributes" >}}

Creating a Source

Create a new source.

Method and URL
POST /api/v3/catalog

Parameters

{{< api-section file="data/software-api/source.json" data="createSourceParameters" >}}

config

{{< api-section file="data/software-api/source.json" data="createConfigAttributes" >}}

The config object's parameters vary for different source types. Read Source Configuration for information about the available parameters in the config object for each supported source type.

metadataPolicy

{{< api-section file="data/software-api/source.json" data="createMetadataPolicyAttributes" >}}

accessControlList

{{< api-section file="data/software-api/source.json" data="createAccessControlListParameters" >}}

users and roles

{{< api-section file="data/software-api/source.json" data="createUsersRolesParameters" >}}


Example Request
curl -X POST 'https://{hostname}/api/v3/catalog' \
--header 'Authorization: _dremio{tokenstring}' \
--header 'Content-Type: application/json' \
--data-raw '{
"entityType": "source",
"config": {
"accessKey": "EXAMPLE78HT89VS4YJEL",
"accessSecret": "EXAMPLEe3bcpKnAwgJ2WBpX8d9kEdhMz24guiR7L",
"secure": true,
"rootPath": "/",
"enableAsync": true,
"compatibilityMode": false,
"isCachingEnabled": true,
"maxCacheSpacePct": 100,
"whitelistedBuckets": [
"archive.dremio.com",
"logs_east-1",
"logs_west-1"
],
"requesterPays": false,
"enableFileStatusCheck": true,
"defaultCtasFormat": "ICEBERG",
"isPartitionInferenceEnabled": false,
"credentialType": "ACCESS_KEY"
},
"type": "S3",
"name": "AWS-S3_testgroup",
"metadataPolicy": {
"authTTLMs": 86400000,
"namesRefreshMs": 3600000,
"datasetRefreshAfterMs": 3600000,
"datasetExpireAfterMs": 10800000,
"datasetUpdateMode": "PREFETCH_QUERIED",
"deleteUnavailableDatasets": true,
"autoPromoteDatasets": false
},
"accelerationGracePeriodMs": 10800000,
"accelerationRefreshPeriodMs": 3600000,
"accelerationNeverExpire": false,
"accelerationNeverRefresh": false,
"allowCrossSourceSelection": false,
"disableMetadataValidityCheck": false,
"accessControlList": {
"users": [
{
"id": "ebe519ab-20e3-43ff-9b4c-b3ec590c7e65",
"permissions": [
"VIEW_REFLECTION",
"SELECT"
]
}
],
"roles": [
{
"id": "c45ff4d8-e910-4f85-89db-9b8c29188a56",
"permissions": [
"ALTER",
"CREATE_TABLE",
"DROP",
"INSERT",
"DELETE",
"UPDATE",
"TRUNCATE",
"VIEW_REFLECTION",
"ALTER_REFLECTION",
"MODIFY",
"MANAGE_GRANTS",
"SELECT"
]
}
]
}
}'
Example Response
{
"entityType": "source",
"config": {
"accessKey": "EXAMPLE78HT89VS4YJEL",
"accessSecret": "$DREMIO_EXISTING_VALUE$",
"secure": true,
"rootPath": "/",
"enableAsync": true,
"compatibilityMode": false,
"isCachingEnabled": true,
"maxCacheSpacePct": 100,
"whitelistedBuckets": [
"archive.dremio.com",
"logs_east-1",
"logs_west-1"
],
"requesterPays": false,
"enableFileStatusCheck": true,
"defaultCtasFormat": "ICEBERG",
"isPartitionInferenceEnabled": false,
"credentialType": "ACCESS_KEY"
},
"state": {
"status": "good",
"suggestedUserAction": "",
"messages": []
},
"id": "2b1be882-7012-4a99-8d6c-82e32e4562e4",
"tag": "T0/Zr1FOY3A=",
"type": "S3",
"name": "AWS-S3_testgroup",
"createdAt": "2023-02-17T14:32:20.640Z",
"metadataPolicy": {
"authTTLMs": 86400000,
"namesRefreshMs": 3600000,
"datasetRefreshAfterMs": 3600000,
"datasetExpireAfterMs": 10800000,
"datasetUpdateMode": "PREFETCH_QUERIED",
"deleteUnavailableDatasets": true,
"autoPromoteDatasets": false
},
"accelerationGracePeriodMs": 10800000,
"accelerationRefreshPeriodMs": 3600000,
"accelerationNeverExpire": false,
"accelerationNeverRefresh": false,
"children": [
{
"id": "dremio:/AWS-S3_testgroup/archive.dremio.com",
"path": [
"AWS-S3_testgroup",
"archive.dremio.com"
],
"type": "CONTAINER",
"containerType": "FOLDER"
},
{
"id": "dremio:/AWS-S3_testgroup/logs_east-1",
"path": [
"AWS-S3_testgroup",
"logs_east-1"
],
"type": "CONTAINER",
"containerType": "FOLDER"
},
{
"id": "dremio:/AWS-S3_testgroup/logs_west-1",
"path": [
"AWS-S3_testgroup",
"logs_west-1"
],
"type": "CONTAINER",
"containerType": "FOLDER"
}
],
"allowCrossSourceSelection": false,
"disableMetadataValidityCheck": false,
"accessControlList": {
"users": [
{
"id": "ebe519ab-20e3-43ff-9b4c-b3ec590c7e65",
"permissions": [
"VIEW_REFLECTION",
"SELECT"
]
}
],
"roles": [
{
"id": "c45ff4d8-e910-4f85-89db-9b8c29188a56",
"permissions": [
"ALTER",
"CREATE_TABLE",
"DROP",
"INSERT",
"DELETE",
"UPDATE",
"TRUNCATE",
"VIEW_REFLECTION",
"ALTER_REFLECTION",
"MODIFY",
"MANAGE_GRANTS",
"SELECT"
]
}
]
},
"permissions": [],
"checkTableAuthorizer": true,
"owner": {
"ownerId": "4fb93af3-acc2-4b10-ad4b-64dd7070d365",
"ownerType": "USER"
}
}

When you use the Catalog API to create a new source, the response includes a state object that describes the status of the source as shown in the example response above. The state object contains the following attributes:

{{< api-section file="data/software-api/source.json" data="createSourceStateResponse" >}}

Response Status Codes

{{< api-responses file="data/software-api/source.json" data="createSourceResponseCodes" >}}

Retrieving a Source by ID

Retrieve a source and information about its contents by specifying the source's ID.

Method and URL
GET /api/v3/catalog/{id}

Parameters

{{< api-section file="data/software-api/source.json" data="retrieveSourceIdParameters" >}}


Example Request
curl -X GET 'https://{hostname}/api/v3/catalog/2b1be882-7012-4a99-8d6c-82e32e4562e4' \
--header 'Authorization: _dremio{tokenstring}' \
--header 'Content-Type: application/json'
Example Response
{
"entityType": "source",
"config": {
"accessKey": "EXAMPLE78HT89VS4YJEL",
"accessSecret": "$DREMIO_EXISTING_VALUE$",
"secure": true,
"rootPath": "/",
"enableAsync": true,
"compatibilityMode": false,
"isCachingEnabled": true,
"maxCacheSpacePct": 100,
"whitelistedBuckets": [
"archive.dremio.com",
"logs_east-1",
"logs_west-1"
],
"requesterPays": false,
"enableFileStatusCheck": true,
"defaultCtasFormat": "ICEBERG",
"isPartitionInferenceEnabled": false,
"credentialType": "ACCESS_KEY"
},
"id": "2b1be882-7012-4a99-8d6c-82e32e4562e4",
"tag": "T0/Zr1FOY3A=",
"type": "S3",
"name": "AWS-S3_testgroup",
"createdAt": "2023-02-17T14:32:20.640Z",
"metadataPolicy": {
"authTTLMs": 86400000,
"namesRefreshMs": 3600000,
"datasetRefreshAfterMs": 3600000,
"datasetExpireAfterMs": 10800000,
"datasetUpdateMode": "PREFETCH_QUERIED",
"deleteUnavailableDatasets": true,
"autoPromoteDatasets": false
},
"accelerationGracePeriodMs": 10800000,
"accelerationRefreshPeriodMs": 3600000,
"accelerationNeverExpire": false,
"accelerationNeverRefresh": false,
"children": [
{
"id": "dremio:/AWS-S3_testgroup/archive.dremio.com",
"path": [
"AWS-S3_testgroup",
"archive.dremio.com"
],
"type": "CONTAINER",
"containerType": "FOLDER"
},
{
"id": "dremio:/AWS-S3_testgroup/logs_east-1",
"path": [
"AWS-S3_testgroup",
"logs_east-1"
],
"type": "CONTAINER",
"containerType": "FOLDER"
},
{
"id": "dremio:/AWS-S3_testgroup/logs_west-1",
"path": [
"AWS-S3_testgroup",
"logs_west-1"
],
"type": "CONTAINER",
"containerType": "FOLDER"
}
],
"allowCrossSourceSelection": false,
"disableMetadataValidityCheck": false,
"accessControlList": {
"users": [
{
"id": "ebe519ab-20e3-43ff-9b4c-b3ec590c7e65",
"permissions": [
"VIEW_REFLECTION",
"SELECT"
]
}
],
"roles": [
{
"id": "c45ff4d8-e910-4f85-89db-9b8c29188a56",
"permissions": [
"ALTER",
"CREATE_TABLE",
"DROP",
"INSERT",
"DELETE",
"UPDATE",
"TRUNCATE",
"VIEW_REFLECTION",
"ALTER_REFLECTION",
"MODIFY",
"MANAGE_GRANTS",
"SELECT"
]
}
]
},
"permissions": [],
"checkTableAuthorizer": true,
"owner": {
"ownerId": "4fb93af3-acc2-4b10-ad4b-64dd7070d365",
"ownerType": "USER"
}
}

Response Status Codes

{{< api-responses file="data/software-api/source.json" data="retrieveSourceIdResponseCodes" >}}

Retrieving a Source by Path

Retrieve a source and information about its contents by specifying the source's path.

Method and URL
GET /api/v3/catalog/by-path/{path}

Parameters

{{< api-section file="data/software-api/source.json" data="retrieveSourcePathParameters" >}}


Example Request
curl -X GET 'https://{hostname}/api/v3/catalog/by-path/AWS-S3_testgroup' \
--header 'Authorization: _dremio{tokenstring}' \
--header 'Content-Type: application/json'
Example Response
{
"entityType": "source",
"config": {
"accessKey": "EXAMPLE78HT89VS4YJEL",
"accessSecret": "$DREMIO_EXISTING_VALUE$",
"secure": true,
"rootPath": "/",
"enableAsync": true,
"compatibilityMode": false,
"isCachingEnabled": true,
"maxCacheSpacePct": 100,
"whitelistedBuckets": [
"archive.dremio.com",
"logs_east-1",
"logs_west-1"
],
"requesterPays": false,
"enableFileStatusCheck": true,
"defaultCtasFormat": "ICEBERG",
"isPartitionInferenceEnabled": false,
"credentialType": "ACCESS_KEY"
},
"id": "2b1be882-7012-4a99-8d6c-82e32e4562e4",
"tag": "T0/Zr1FOY3A=",
"type": "S3",
"name": "AWS-S3_testgroup",
"createdAt": "2023-02-17T14:32:20.640Z",
"metadataPolicy": {
"authTTLMs": 86400000,
"namesRefreshMs": 3600000,
"datasetRefreshAfterMs": 3600000,
"datasetExpireAfterMs": 10800000,
"datasetUpdateMode": "PREFETCH_QUERIED",
"deleteUnavailableDatasets": true,
"autoPromoteDatasets": false
},
"accelerationGracePeriodMs": 10800000,
"accelerationRefreshPeriodMs": 3600000,
"accelerationNeverExpire": false,
"accelerationNeverRefresh": false,
"children": [
{
"id": "dremio:/AWS-S3_testgroup/archive.dremio.com",
"path": [
"AWS-S3_testgroup",
"archive.dremio.com"
],
"type": "CONTAINER",
"containerType": "FOLDER"
},
{
"id": "dremio:/AWS-S3_testgroup/logs_east-1",
"path": [
"AWS-S3_testgroup",
"logs_east-1"
],
"type": "CONTAINER",
"containerType": "FOLDER"
},
{
"id": "dremio:/AWS-S3_testgroup/logs_west-1",
"path": [
"AWS-S3_testgroup",
"logs_west-1"
],
"type": "CONTAINER",
"containerType": "FOLDER"
}
],
"allowCrossSourceSelection": false,
"disableMetadataValidityCheck": false,
"accessControlList": {
"users": [
{
"id": "ebe519ab-20e3-43ff-9b4c-b3ec590c7e65",
"permissions": [
"VIEW_REFLECTION",
"SELECT"
]
}
],
"roles": [
{
"id": "c45ff4d8-e910-4f85-89db-9b8c29188a56",
"permissions": [
"ALTER",
"CREATE_TABLE",
"DROP",
"INSERT",
"DELETE",
"UPDATE",
"TRUNCATE",
"VIEW_REFLECTION",
"ALTER_REFLECTION",
"MODIFY",
"MANAGE_GRANTS",
"SELECT"
]
}
]
},
"permissions": [],
"checkTableAuthorizer": true,
"owner": {
"ownerId": "4fb93af3-acc2-4b10-ad4b-64dd7070d365",
"ownerType": "USER"
}
}

Response Status Codes

{{< api-responses file="data/software-api/source.json" data="retrieveSourcePathResponseCodes" >}}

Updating a Source

Update the specified source.

Method and URL
PUT /api/v3/catalog/{id}

Parameters

{{< api-section file="data/software-api/source.json" data="updateSourceParameters" >}}

config

{{< api-section file="data/software-api/source.json" data="updateConfigAttributes" >}}

The config object's parameters vary for different source types. Read Source Configuration for information about the available parameters in the config object for each supported source type.

metadataPolicy

{{< api-section file="data/software-api/source.json" data="updateMetadataPolicyAttributes" >}}

accessControlList

{{< api-section file="data/software-api/source.json" data="updateAccessControlListParameters" >}}

users and roles

{{< api-section file="data/software-api/source.json" data="updateUsersRolesParameters" >}}


Example Request
curl -X PUT 'https://{hostname}/api/v3/catalog/2b1be882-7012-4a99-8d6c-82e32e4562e4' \
--header 'Authorization: _dremio{tokenstring}' \
--header 'Content-Type: application/json' \
--data-raw '{
"entityType": "source",
"config": {
"accessKey": "EXAMPLE78HT89VS4YJEL",
"accessSecret": "$DREMIO_EXISTING_VALUE$",
"secure": true,
"rootPath": "/",
"enableAsync": true,
"compatibilityMode": false,
"isCachingEnabled": true,
"maxCacheSpacePct": 100,
"whitelistedBuckets": [
"archive.dremio.com",
"logs_east-1",
"logs_west-1"
],
"requesterPays": false,
"enableFileStatusCheck": true,
"defaultCtasFormat": "ICEBERG",
"isPartitionInferenceEnabled": false,
"credentialType": "ACCESS_KEY"
},
"id": "2b1be882-7012-4a99-8d6c-82e32e4562e4",
"tag": "T0/Zr1FOY3A=",
"type": "S3",
"name": "AWS-S3_testgroup",
"metadataPolicy": {
"authTTLMs": 86400000,
"namesRefreshMs": 3600000,
"datasetRefreshAfterMs": 3600000,
"datasetExpireAfterMs": 10800000,
"datasetUpdateMode": "PREFETCH_QUERIED",
"deleteUnavailableDatasets": true,
"autoPromoteDatasets": true
},
"accelerationGracePeriodMs": 10800000,
"accelerationRefreshPeriodMs": 3600000,
"accelerationNeverExpire": false,
"accelerationNeverRefresh": false,
"allowCrossSourceSelection": false,
"disableMetadataValidityCheck": false,
"accessControlList": {
"users": [
{
"id": "ebe519ab-20e3-43ff-9b4c-b3ec590c7e65",
"permissions": [
"ALTER",
"VIEW_REFLECTION",
"SELECT"
]
}
],
"roles": [
{
"id": "c45ff4d8-e910-4f85-89db-9b8c29188a56",
"permissions": [
"ALTER",
"CREATE_TABLE",
"DROP",
"INSERT",
"DELETE",
"UPDATE",
"TRUNCATE",
"VIEW_REFLECTION",
"ALTER_REFLECTION",
"MODIFY",
"MANAGE_GRANTS",
"SELECT"
]
}
]
}
}'
Example Response
{
"entityType": "source",
"config": {
"accessKey": "EXAMPLE78HT89VS4YJEL",
"accessSecret": "$DREMIO_EXISTING_VALUE$",
"secure": true,
"rootPath": "/",
"enableAsync": true,
"compatibilityMode": false,
"isCachingEnabled": true,
"maxCacheSpacePct": 100,
"whitelistedBuckets": [
"archive.dremio.com",
"logs_east-1",
"logs_west-1"
],
"requesterPays": false,
"enableFileStatusCheck": true,
"defaultCtasFormat": "ICEBERG",
"isPartitionInferenceEnabled": false,
"credentialType": "ACCESS_KEY"
},
"id": "2b1be882-7012-4a99-8d6c-82e32e4562e4",
"tag": "RfVMBBMWRvU=",
"type": "S3",
"name": "AWS-S3_testgroup",
"createdAt": "2023-02-17T14:32:20.640Z",
"metadataPolicy": {
"authTTLMs": 86400000,
"namesRefreshMs": 3600000,
"datasetRefreshAfterMs": 3600000,
"datasetExpireAfterMs": 10800000,
"datasetUpdateMode": "PREFETCH_QUERIED",
"deleteUnavailableDatasets": true,
"autoPromoteDatasets": true
},
"accelerationGracePeriodMs": 10800000,
"accelerationRefreshPeriodMs": 3600000,
"accelerationNeverExpire": false,
"accelerationNeverRefresh": false,
"children": [
{
"id": "dremio:/AWS-S3_testgroup/archive.dremio.com",
"path": [
"AWS-S3_testgroup",
"archive.dremio.com"
],
"type": "CONTAINER",
"containerType": "FOLDER"
},
{
"id": "dremio:/AWS-S3_testgroup/logs_east-1",
"path": [
"AWS-S3_testgroup",
"logs_east-1"
],
"type": "CONTAINER",
"containerType": "FOLDER"
},
{
"id": "dremio:/AWS-S3_testgroup/logs_west-1",
"path": [
"AWS-S3_testgroup",
"logs_west-1"
],
"type": "CONTAINER",
"containerType": "FOLDER"
}
],
"allowCrossSourceSelection": false,
"disableMetadataValidityCheck": false,
"accessControlList": {
"users": [
{
"id": "ebe519ab-20e3-43ff-9b4c-b3ec590c7e65",
"permissions": [
"ALTER",
"VIEW_REFLECTION",
"SELECT"
]
}
],
"roles": [
{
"id": "c45ff4d8-e910-4f85-89db-9b8c29188a56",
"permissions": [
"ALTER",
"CREATE_TABLE",
"DROP",
"INSERT",
"DELETE",
"UPDATE",
"TRUNCATE",
"VIEW_REFLECTION",
"ALTER_REFLECTION",
"MODIFY",
"MANAGE_GRANTS",
"SELECT"
]
}
]
},
"permissions": [],
"checkTableAuthorizer": true,
"owner": {
"ownerId": "4fb93af3-acc2-4b10-ad4b-64dd7070d365",
"ownerType": "USER"
}
}

Deleting a Source

Delete the specified source, including all of the source's contents.

Method and URL
DELETE /api/v3/catalog/{id}

Parameters

{{< api-section file="data/software-api/source.json" data="deleteSourceParameters" >}}


Example Request
curl -X DELETE 'https://{hostname}/api/v3/catalog/2b1be882-7012-4a99-8d6c-82e32e4562e4' \
--header 'Authorization: _dremio{tokenstring}' \
--header 'Content-Type: application/json'
Example Response
No response

Response Status Codes

{{< api-responses file="data/software-api/source.json" data="deleteSourceResponseCodes" >}}