Skip to main content

Source

The Source API enables you to manage the sources that are supported in Dremio. You can add, retrieve, update, and delete sources. Sources are of entityType: source.

note:

The source object contains the same attributes for all source types except for the config object, which is different for each source type. The examples on this page use an Amazon S3 source to demonstrate the available endpoints. For configuration information for each source, see Configuration Attributes.

Source Object for Amazon S3
{
  "entityType": "source",
  "config": {
    "accessKey": "AKIAIOSFODNN7EXAMPLE",
    "accessSecret": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
    "secure": true,
    "whitelistedBuckets": [
        "sales",
        "marketing"
    ],
    "rootPath": "/",
    "enableAsync": true,
    "compatibilityMode": false,
    "isCachingEnabled": true,
    "maxCacheSpacePct": 100,
    "requesterPays": false,
    "enableFileStatusCheck": true,
    "defaultCtasFormat": "ICEBERG",
    "isPartitionInferenceEnabled": false,
    "credentialType": "ACCESS_KEY"
  },  
  "id": "4abb1367-5282-43e8-a0f6-54a8ae6428ed",
  "tag": "676e73e0-8b51-44d0-972d-ef80c29ac0c6",
  "type": "S3",
  "name": "source-name",
  "createdAt": "2022-07-26T19:56:28.826Z",
  "metadataPolicy": {
    "authTTLMs": 86400000,
    "namesRefreshMs": 3600000,
    "datasetRefreshAfterMs": 3600000,
    "datasetExpireAfterMs": 10800000,
    "datasetUpdateMode": "PREFETCH_QUERIED",
    "deleteUnavailableDatasets": true,
    "autoPromoteDatasets": false
  },
  "accelerationGracePeriodMs": 10800000,
  "accelerationRefreshPeriodMs": 3600000,
  "accelerationNeverExpire": false,
  "accelerationNeverRefresh": false,
  "children": [
    {
      "id": "dremio:/customer-records/sales",
      "path": [
          "customer-records",
          "sales"
      ],
      "type": "CONTAINER",
      "containerType": "FOLDER"
  },
  {
      "id": "dremio:/customer-records/marketing",
      "path": [
          "customer-records",
          "marketing"
      ],
      "type": "CONTAINER",
      "containerType": "FOLDER"
  },
],
  "allowCrossSourceSelection": false,
  "disableMetadataValidityCheck": false,
  "accessControlList": {
     "roles": [
       {
         "id": "1e5f38e4-8209-46dc-96f0-cfbd3276dbd8",
         "permissions": [
             "ALTER",
             "CREATE_TABLE",
             "DROP",
             "INSERT",
             "DELETE",
             "UPDATE",
             "TRUNCATE",
             "VIEW_REFLECTION",
             "ALTER_REFLECTION",
             "MODIFY",
             "MANAGE_GRANTS",
             "SELECT"
           ]
        }
    ],
    "users": [
      {
        "id": "4deaf0e4-f05c-441c-bb37-d4d6e43fdec7",
        "permissions": [
            "VIEW_REFLECTION",
            "SELECT"
        ]
      }
    ]
  },
  "permissions": [
    "READ",
    "WRITE",
    "ALTER_REFLECTION",
    "SELECT",
    "ALTER",
    "VIEW_REFLECTION",
    "MODIFY",
    "MANAGE_GRANTS",
    "CREATE_TABLE",
    "DROP",
    "EXTERNAL_QUERY",
    "INSERT",
    "TRUNCATE",
    "DELETE",
    "UPDATE",
    "EXECUTE",
    "CREATE_SOURCE",
    "ALL"
  ],
  "owner": {
    "ownerId": "5c9c394d-7395-4687-97dd-beac4384c359",
    "ownerType": "USER"
  }
}

Source Attributes​

{{< api-section file="data/api/source.json" data="sourceAttributes" >}}

config​

{{< api-section file="data/api/source.json" data="config" >}}

metadataPolicy​

{{< api-section file="data/api/source.json" data="metadataPolicy" >}}

children​

{{< api-section file="data/api/source.json" data="children" >}}

accessControlList​

{{< api-section file="data/api/source.json" data="accessControlList" >}}

roles​

{{< api-section file="data/api/source.json" data="roles" >}}

users​

{{< api-section file="data/api/source.json" data="users" >}}

owner​

{{< api-section file="data/api/source.json" data="owner" >}}

Adding a Source​

Add a source.

Adding a Source
POST  /v0/projects/{project-id}/catalog

Parameters​

{{< api-section file="data/api/source.json" data="addingSourceParameters" >}}

config​

{{< api-section file="data/api/source.json" data="config-1" >}}

metadataPolicy​

{{< api-section file="data/api/source.json" data="metadataPolicy-1" >}}

accessControlList​

{{< api-section file="data/api/source.json" data="accessControlList-1" >}}

roles​

{{< api-section file="data/api/source.json" data="roles-1" >}}

users​

{{< api-section file="data/api/source.json" data="users-1" >}}

Example Request
curl -X POST 'https://api.dremio.cloud/v0/projects/02d36975-73eb-47ed-9bb5-de73060380f6/catalog' \
-H 'Authorization: Bearer <personal access token>' \
-H 'Content-Type: application/json' \
-d '{
      "entityType": "source",
      "config": {
        "hostname": "177.15.0.112",
        "port": "3306",
        "authenticationType": "ANONYMOUS",
        "netWriteTimeout": 60,
        "fetchSize": 200,
        "maxIdleConns": 8,
        "idleTimeSec": 60,
        "propertyList": []
      },
      "type": "MYSQL",
      "name": "recruitingdb",
      "metadataPolicy": {
        "authTTLMs": 86400000,
        "namesRefreshMs": 86400000,
        "datasetRefreshAfterMs": 86400000,
        "datasetExpireAfterMs": 259200000,
        "datasetUpdateMode": "PREFETCH_QUERIED",
        "deleteUnavailableDatasets": true
      },
      "accelerationGracePeriodMs": 10800000,
      "accelerationRefreshPeriodMs": 3600000,
      "accessControlList": {
        "roles": [
          {
            "id": "9ab42853-bdef-465f-b9bb-381a13a9bf78",
            "permissions": []
          }
        ],
        "users": []
      }
    }'
Example Response
{
  "entityType": "source",
  "config": {
    "hostname": "177.15.0.112",
    "port": "3306",
    "authenticationType": "ANONYMOUS",
    "netWriteTimeout": 60,
    "fetchSize": 200,
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "propertyList": []
  },
  "id": "629904cf-9c06-4ae6-8cc1-faf1d2f1ab5f",
  "tag": "a927dd14-2c7d-4684-9de2-4000e05c0d07",
  "type": "MYSQL",
  "name": "recruitingdb",
  "createdAt": "2022-09-12T07:22:19.334Z",
  "metadataPolicy": {
    "authTTLMs": 86400000,
    "namesRefreshMs": 86400000,
    "datasetRefreshAfterMs": 86400000,
    "datasetExpireAfterMs": 259200000,
    "datasetUpdateMode": "PREFETCH_QUERIED",
    "deleteUnavailableDatasets": true
  },
  "accelerationGracePeriodMs": 10800000,
  "accelerationRefreshPeriodMs": 3600000,
  "accelerationNeverExpire": false,
  "accelerationNeverRefresh": false,
  "children": [
      {
        "id": "42312a90-1723-4bcb-b62c-bf9abee371b1",
        "path": [
            "recruitingdb",
            "contacts"
          ],
          "type": "CONTAINER",
          "containerType": "FOLDER"
      }
  ],
  "allowCrossSourceSelection": false,
  "disableMetadataValidityCheck": false,
  "accessControlList": {
      "roles": [
        {
            "id": "9ab42853-bdef-465f-b9bb-381a13a9bf78",
            "permissions": []
        }
      ],
      "users": []
  },
  "permissions": [],
  "checkTableAuthorizer": false,
  "owner": {
      "ownerId": "5c9c394d-7395-4687-97dd-beac4384c359",
      "ownerType": "USER"
  }
}

Responses​

{{< api-responses file="data/api/source.json" data="addingSourceResponses" >}}

Retrieving a Source​

Get information about the source using the source ID.

Retrieving a Source
GET /v0/projects/{project-id}/catalog/{id}

Parameters​

{{< api-section file="data/api/source.json" data="retrievingSourceParameters" >}}

Example Request
curl -X GET 'https://api.dremio.cloud/v0/projects/02d36975-73eb-47ed-9bb5-de73060380f6/catalog/ffbe8c1d-1db7-48d1-9c58-f452838fedc0' \
-H 'Authorization: Bearer <personal access token>' \
-H 'Content-Type: application/json'
Example Response
{
  "entityType": "source",
  "config": {
    "hostname": "177.15.0.112",
    "port": "3306",
    "authenticationType": "ANONYMOUS",
    "netWriteTimeout": 60,
    "fetchSize": 200,
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "propertyList": []
  },
  "id": "629904cf-9c06-4ae6-8cc1-faf1d2f1ab5f",
  "tag": "a927dd14-2c7d-4684-9de2-4000e05c0d07",
  "type": "MYSQL",
  "name": "recruitingdb",
  "createdAt": "2022-09-12T07:22:19.334Z",
  "metadataPolicy": {
    "authTTLMs": 86400000,
    "namesRefreshMs": 86400000,
    "datasetRefreshAfterMs": 86400000,
    "datasetExpireAfterMs": 259200000,
    "datasetUpdateMode": "PREFETCH_QUERIED",
    "deleteUnavailableDatasets": true
  },
  "accelerationGracePeriodMs": 10800000,
  "accelerationRefreshPeriodMs": 3600000,
  "accelerationNeverExpire": false,
  "accelerationNeverRefresh": false,
  "children": [
    {
      "id": "42312a90-1723-4bcb-b62c-bf9abee371b1",
      "path": [
        "recruitingdb",
        "contacts"
      ],
      "type": "CONTAINER",
      "containerType": "FOLDER"
    }
  ],
  "allowCrossSourceSelection": false,
  "disableMetadataValidityCheck": false,
  "accessControlList": {},
  "permissions": [],
  "checkTableAuthorizer": false,
  "owner": {
    "ownerId": "5c9c394d-7395-4687-97dd-beac4384c359",
    "ownerType": "USER"
  }
}

Responses​

{{< api-responses file="data/api/source.json" data="retrievingSourceResponses" >}}

Updating a Source​

Edit the source information using the source ID.

Updating a Source
PUT /v0/projects/{project-id}/catalog/{id}

Parameters​

{{< api-section file="data/api/source.json" data="updatingSourceParameters" >}}

config​

{{< api-section file="data/api/source.json" data="config-3" >}}

metadataPolicy​

{{< api-section file="data/api/source.json" data="metadataPolicy-3" >}}

accessControlList​

{{< api-section file="data/api/source.json" data="accessControlList-3" >}}

roles​

{{< api-section file="data/api/source.json" data="roles-3" >}}

Example Request
curl -X GET 'https://api.dremio.cloud/v0/projects/02d36975-73eb-47ed-9bb5-de73060380f6/catalog/ffbe8c1d-1db7-48d1-9c58-f452838fedc0' \
-H 'Authorization: Bearer <personal access token>' \
-H 'Content-Type: application/json'
-d '{
      "entityType": "source",
      "config": {
        "hostname": "177.15.0.112",
        "port": "3306",
        "authenticationType": "ANONYMOUS",
        "netWriteTimeout": 60,
        "fetchSize": 200,
        "maxIdleConns": 8,
        "idleTimeSec": 60,
        "propertyList": []
      },
      "type": "MYSQL",
      "name": "recruitingdb",
      "metadataPolicy": {
        "authTTLMs": 86400000,
        "namesRefreshMs": 86400000,
        "datasetRefreshAfterMs": 86400000,
        "datasetExpireAfterMs": 259200000,
        "datasetUpdateMode": "PREFETCH_QUERIED",
        "deleteUnavailableDatasets": true
      },
      "accelerationGracePeriodMs": 10800000,
      "accelerationRefreshPeriodMs": 3600000,
      "accessControlList": {
        "roles": [
          {
            "id": "9ab42853-bdef-465f-b9bb-381a13a9bf78",
            "permissions": []
          }
        ],
        "users": []
      }
    }'
Example Response
{
  "entityType": "source",
  "config": {
    "hostname": "177.15.0.112",
    "port": "3306",
    "authenticationType": "ANONYMOUS",
    "netWriteTimeout": 60,
    "fetchSize": 200,
    "maxIdleConns": 8,
    "idleTimeSec": 60,
    "propertyList": []
  },
  "id": "629904cf-9c06-4ae6-8cc1-faf1d2f1ab5f",
  "tag": "a927dd14-2c7d-4684-9de2-4000e05c0d07",
  "type": "MYSQL",
  "name": "recruitingdb",
  "createdAt": "2022-09-12T07:22:19.334Z",
  "metadataPolicy": {
    "authTTLMs": 86400000,
    "namesRefreshMs": 86400000,
    "datasetRefreshAfterMs": 86400000,
    "datasetExpireAfterMs": 259200000,
    "datasetUpdateMode": "PREFETCH_QUERIED",
    "deleteUnavailableDatasets": true
  },
  "accelerationGracePeriodMs": 10800000,
  "accelerationRefreshPeriodMs": 3600000,
  "accelerationNeverExpire": false,
  "accelerationNeverRefresh": false,
  "children": [
    {
      "id": "42312a90-1723-4bcb-b62c-bf9abee371b1",
      "path": [
          "recruitingdb",
          "contacts"
        ],
        "type": "CONTAINER",
        "containerType": "FOLDER"
    }
  ],
  "allowCrossSourceSelection": false,
  "disableMetadataValidityCheck": false,
  "accessControlList": {
    "roles": [
      {
          "id": "9ab42853-bdef-465f-b9bb-381a13a9bf78",
          "permissions": []
      }
    ],
    "users": []
  },
  "permissions": [],
  "checkTableAuthorizer": false,
  "owner": {
    "ownerId": "5c9c394d-7395-4687-97dd-beac4384c359",
    "ownerType": "USER"
  }
}

Responses​

{{< api-responses file="data/api/source.json" data="updatingSourceResponses" >}}

Deleting a Source​

Delete a source.

Deleting a Source
DELETE /v0/projects/{project-id}/catalog/{id}

Parameters​

{{< api-section file="data/api/source.json" data="deletingSourceParameters" >}}

Example Request
curl -X DELETE 'https://api.dremio.cloud/v0/projects/02d36975-73eb-47ed-9bb5-de73060380f6/catalog/21a74cb1-3fcf-48c6-8f69-94c257ec3da3' \
-H 'Authorization: Bearer <personal access token>' \
-H 'Content-Type: application/json'
Example Response
No response

Responses​

{{< api-responses file="data/api/source.json" data="deletingSourceResponses" >}}