Reflection Summary
Use the Reflection API to retrieve a reflection summary that includes all raw and aggregation reflections for the Dremio instance.
Reflection summary objects are different from reflection objects. Reflection summaries do not include certain attributes that define the reflection, like the display, dimension, measure, sort, and partition attributes. Reflection summaries do include several attributes that do not appear in reflection objects, like datasetType, datasetPath, and counts and links for considered, matched, and chosen jobs.
Reflection Summary Object{
"data": [
{
"createdAt": "2022-07-05T19:19:40.244Z",
"updatedAt": "2023-01-13T19:46:01.313Z",
"id": "27077c03-ae49-454c-a7bb-a9a8b5eca224",
"reflectionType": "AGGREGATION",
"name": "NYC_taxi_agg",
"currentSizeBytes": 9272,
"outputRecords": 51,
"totalSizeBytes": 9272,
"datasetId": "fa7c487f-9550-474e-8a41-4826564c6b09",
"datasetType": "VIRTUAL_DATASET",
"datasetPath": [
"Samples",
"samples.dremio.com",
"NYC-taxi-trips"
],
"status": {
"configStatus": "OK",
"refreshStatus": "MANUAL",
"availabilityStatus": "AVAILABLE",
"combinedStatus": "CAN_ACCELERATE",
"refreshMethod": "FULL",
"failureCount": 0,
"lastDataFetchAt": "2023-01-13T19:05:03.532Z",
"expiresAt": "3022-05-16T19:46:02.342Z",
"lastRefreshDurationMillis": 46387
},
"consideredCount": 202,
"matchedCount": 45,
"chosenCount": 5,
"consideredJobsLink": "/jobs?filters=%7B%22cor%22%3A%5B%22422ace5b8a9a-bb7a-c454-94ea-30c77072%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"matchedJobsLink": "/jobs?filters=%7B%22mar%22%3A%5B%22422ace5b8a9a-bb7a-c454-94ea-30c77072%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"chosenJobsLink": "/jobs?filters=%7B%22chr%22%3A%5B%22422ace5b8a9a-bb7a-c454-94ea-30c77072%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"isArrowCachingEnabled": false,
"isCanView": true,
"isCanAlter": true,
"isEnabled": true
},
{
"createdAt": "2023-01-26T23:27:04.281Z",
"updatedAt": "2023-01-26T23:27:04.281Z",
"id": "0e3d765a-2291-4a04-81eb-2daf5477cc7d",
"reflectionType": "RAW",
"name": "Raw Reflection",
"currentSizeBytes": 0,
"outputRecords": -1,
"totalSizeBytes": 0,
"datasetId": "acdad4be-7049-47e4-b616-b471c5b3c60c",
"datasetType": "PHYSICAL_DATASET",
"datasetPath": [
"@dremio",
"test"
],
"status": {
"configStatus": "OK",
"refreshStatus": "GIVEN_UP",
"availabilityStatus": "NONE",
"combinedStatus": "FAILED",
"refreshMethod": "NONE",
"failureCount": 3,
"lastDataFetchAt": null,
"expiresAt": null,
"lastRefreshDurationMillis": -1
},
"consideredCount": 0,
"matchedCount": 0,
"chosenCount": 0,
"consideredJobsLink": "/jobs?filters=%7B%22cor%22%3A%5B%22d7cc7745fad2-be18-40a4-1922-a567d3e0%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"matchedJobsLink": "/jobs?filters=%7B%22mar%22%3A%5B%22d7cc7745fad2-be18-40a4-1922-a567d3e0%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"chosenJobsLink": "/jobs?filters=%7B%22chr%22%3A%5B%22d7cc7745fad2-be18-40a4-1922-a567d3e0%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"isArrowCachingEnabled": false,
"isCanView": true,
"isCanAlter": true,
"isEnabled": true
},
{
"createdAt": "2023-01-13T19:50:19.030Z",
"updatedAt": "2023-01-13T19:50:19.030Z",
"id": "8eec62d7-3419-4cf3-997d-0a153d81ed8a",
"reflectionType": "AGGREGATION",
"name": "dataset991_agg991",
"currentSizeBytes": 9273,
"outputRecords": 51,
"totalSizeBytes": 9273,
"datasetId": "a461bf97-8464-43ed-bd86-a8fb90d920e3",
"datasetType": "VIRTUAL_DATASET",
"datasetPath": [
"temp",
"dataset991"
],
"status": {
"configStatus": "OK",
"refreshStatus": "MANUAL",
"availabilityStatus": "AVAILABLE",
"combinedStatus": "CAN_ACCELERATE",
"refreshMethod": "FULL",
"failureCount": 0,
"lastDataFetchAt": "2023-01-13T19:46:02.342Z",
"expiresAt": "3022-05-16T19:46:02.342Z",
"lastRefreshDurationMillis": 11697
},
"consideredCount": 60,
"matchedCount": 9,
"chosenCount": 0,
"consideredJobsLink": "/jobs?filters=%7B%22cor%22%3A%5B%22a8de18d351a0-d799-3fc4-9143-7d26cee8%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"matchedJobsLink": "/jobs?filters=%7B%22mar%22%3A%5B%22a8de18d351a0-d799-3fc4-9143-7d26cee8%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"chosenJobsLink": "/jobs?filters=%7B%22chr%22%3A%5B%22a8de18d351a0-d799-3fc4-9143-7d26cee8%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"isArrowCachingEnabled": false,
"isCanView": true,
"isCanAlter": true,
"isEnabled": true
},
{
"createdAt": "2023-01-13T19:50:17.714Z",
"updatedAt": "2023-01-13T19:50:17.714Z",
"id": "167428eb-7936-4ea2-a1fb-23b1ac6e9454",
"reflectionType": "RAW",
"name": "dataset991_raw991",
"currentSizeBytes": 818790,
"outputRecords": 29467,
"totalSizeBytes": 818790,
"datasetId": "a461bf97-8464-43ed-bd86-a8fb90d920e3",
"datasetType": "VIRTUAL_DATASET",
"datasetPath": [
"temp",
"dataset991"
],
"status": {
"configStatus": "OK",
"refreshStatus": "MANUAL",
"availabilityStatus": "AVAILABLE",
"combinedStatus": "CAN_ACCELERATE",
"refreshMethod": "FULL",
"failureCount": 0,
"lastDataFetchAt": "2023-01-13T19:46:02.747Z",
"expiresAt": "3022-05-16T19:46:02.747Z",
"lastRefreshDurationMillis": 16666
},
"consideredCount": 54,
"matchedCount": 37,
"chosenCount": 0,
"consideredJobsLink": "/jobs?filters=%7B%22cor%22%3A%5B%224549e6ca1b32-bf1a-2ae4-6397-be824761%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"matchedJobsLink": "/jobs?filters=%7B%22mar%22%3A%5B%224549e6ca1b32-bf1a-2ae4-6397-be824761%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"chosenJobsLink": "/jobs?filters=%7B%22chr%22%3A%5B%224549e6ca1b32-bf1a-2ae4-6397-be824761%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"isArrowCachingEnabled": false,
"isCanView": true,
"isCanAlter": true,
"isEnabled": true
}
],
"nextPageToken": "CiQxNjc0MjhlYi03OTM2LTRlYTItYTFmYi0yM2IxYWM2ZTk0NTQSAA==",
"isCanAlterReflections": true
}
Reflection Summary Attributes
{{< api-section file="data/api/reflection.json" data="reflectionSummaryAttributes" >}}
data
{{< api-section file="data/api/reflection.json" data="dataSummaryAttributes" >}}
status
{{< api-section file="data/api/reflection.json" data="statusSummaryAttributes" >}}
Retrieve a Reflection Summary
Retrieve a summary of all raw and aggregation reflections in the Dremio instance.
Method and URLGET /v0/api/projects/{project-id}/reflection-summary
Parameters
{{< api-section file="data/api/reflection.json" data="reflectionSummaryParameters" >}}
Example Request Without Query Parameters
curl -X GET 'https://api.dremio.cloud/v0/api/projects/{project-id}/reflection-summary'
--header 'Authorization: Bearer <PersonalAccessToken>' \
--header "Content-Type: application/json"
{
"data": [
{
"createdAt": "2023-01-13T19:46:01.313Z",
"updatedAt": "2023-01-13T19:46:01.313Z",
"id": "27077c03-ae49-454c-a7bb-a9a8b5eca224",
"reflectionType": "AGGREGATION",
"name": "NYC_taxi_agg",
"currentSizeBytes": 9272,
"outputRecords": 51,
"totalSizeBytes": 9272,
"datasetId": "fa7c487f-9550-474e-8a41-4826564c6b09",
"datasetType": "VIRTUAL_DATASET",
"datasetPath": [
"Samples",
"samples.dremio.com",
"NYC-taxi-trips"
],
"status": {
"configStatus": "OK",
"refreshStatus": "MANUAL",
"availabilityStatus": "AVAILABLE",
"combinedStatus": "CAN_ACCELERATE",
"refreshMethod": "FULL",
"failureCount": 0,
"lastDataFetchAt": "2023-01-13T19:46:02.342Z",
"expiresAt": "3022-05-16T19:46:02.342Z",
"lastRefreshDurationMillis": 46387
},
"consideredCount": 202,
"matchedCount": 45,
"chosenCount": 5,
"consideredJobsLink": "/jobs?filters=%7B%22cor%22%3A%5B%22422ace5b8a9a-bb7a-c454-94ea-30c77072%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"matchedJobsLink": "/jobs?filters=%7B%22mar%22%3A%5B%22422ace5b8a9a-bb7a-c454-94ea-30c77072%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"chosenJobsLink": "/jobs?filters=%7B%22chr%22%3A%5B%22422ace5b8a9a-bb7a-c454-94ea-30c77072%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"isArrowCachingEnabled": false,
"isCanView": true,
"isCanAlter": true,
"isEnabled": true
},
{
"createdAt": "2023-01-26T23:27:04.281Z",
"updatedAt": "2023-01-26T23:27:04.281Z",
"id": "0e3d765a-2291-4a04-81eb-2daf5477cc7d",
"reflectionType": "RAW",
"name": "Raw Reflection",
"currentSizeBytes": 0,
"outputRecords": -1,
"totalSizeBytes": 0,
"datasetId": "acdad4be-7049-47e4-b616-b471c5b3c60c",
"datasetType": "PHYSICAL_DATASET",
"datasetPath": [
"@dremio",
"test"
],
"status": {
"configStatus": "OK",
"refreshStatus": "GIVEN_UP",
"availabilityStatus": "NONE",
"combinedStatus": "FAILED",
"refreshMethod": "NONE",
"failureCount": 3,
"lastDataFetchAt": null,
"expiresAt": null,
"lastRefreshDurationMillis": -1
},
"consideredCount": 0,
"matchedCount": 0,
"chosenCount": 0,
"consideredJobsLink": "/jobs?filters=%7B%22cor%22%3A%5B%22d7cc7745fad2-be18-40a4-1922-a567d3e0%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"matchedJobsLink": "/jobs?filters=%7B%22mar%22%3A%5B%22d7cc7745fad2-be18-40a4-1922-a567d3e0%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"chosenJobsLink": "/jobs?filters=%7B%22chr%22%3A%5B%22d7cc7745fad2-be18-40a4-1922-a567d3e0%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"isArrowCachingEnabled": false,
"isCanView": true,
"isCanAlter": true,
"isEnabled": true
},
{
"createdAt": "2023-01-13T19:50:19.030Z",
"updatedAt": "2023-01-13T19:50:19.030Z",
"id": "8eec62d7-3419-4cf3-997d-0a153d81ed8a",
"reflectionType": "AGGREGATION",
"name": "dataset991_agg991",
"currentSizeBytes": 9273,
"outputRecords": 51,
"totalSizeBytes": 9273,
"datasetId": "a461bf97-8464-43ed-bd86-a8fb90d920e3",
"datasetType": "VIRTUAL_DATASET",
"datasetPath": [
"temp",
"dataset991"
],
"status": {
"configStatus": "OK",
"refreshStatus": "MANUAL",
"availabilityStatus": "AVAILABLE",
"combinedStatus": "CAN_ACCELERATE",
"refreshMethod": "FULL",
"failureCount": 0,
"lastDataFetchAt": "2023-01-13T19:46:02.342Z",
"expiresAt": "3022-05-16T19:46:02.342Z",
"lastRefreshDurationMillis": 11697
},
"consideredCount": 60,
"matchedCount": 9,
"chosenCount": 0,
"consideredJobsLink": "/jobs?filters=%7B%22cor%22%3A%5B%22a8de18d351a0-d799-3fc4-9143-7d26cee8%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"matchedJobsLink": "/jobs?filters=%7B%22mar%22%3A%5B%22a8de18d351a0-d799-3fc4-9143-7d26cee8%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"chosenJobsLink": "/jobs?filters=%7B%22chr%22%3A%5B%22a8de18d351a0-d799-3fc4-9143-7d26cee8%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"isArrowCachingEnabled": false,
"isCanView": true,
"isCanAlter": true,
"isEnabled": true
},
{
"createdAt": "2023-01-13T19:50:17.714Z",
"updatedAt": "2023-01-13T19:50:17.714Z",
"id": "167428eb-7936-4ea2-a1fb-23b1ac6e9454",
"reflectionType": "RAW",
"name": "dataset991_raw991",
"currentSizeBytes": 818790,
"outputRecords": 29467,
"totalSizeBytes": 818790,
"datasetId": "a461bf97-8464-43ed-bd86-a8fb90d920e3",
"datasetType": "VIRTUAL_DATASET",
"datasetPath": [
"temp",
"dataset991"
],
"status": {
"configStatus": "OK",
"refreshStatus": "MANUAL",
"availabilityStatus": "AVAILABLE",
"combinedStatus": "CAN_ACCELERATE",
"refreshMethod": "FULL",
"failureCount": 0,
"lastDataFetchAt": "2023-01-13T19:46:02.747Z",
"expiresAt": "3022-05-16T19:46:02.747Z",
"lastRefreshDurationMillis": 16666
},
"consideredCount": 54,
"matchedCount": 37,
"chosenCount": 0,
"consideredJobsLink": "/jobs?filters=%7B%22cor%22%3A%5B%224549e6ca1b32-bf1a-2ae4-6397-be824761%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"matchedJobsLink": "/jobs?filters=%7B%22mar%22%3A%5B%224549e6ca1b32-bf1a-2ae4-6397-be824761%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"chosenJobsLink": "/jobs?filters=%7B%22chr%22%3A%5B%224549e6ca1b32-bf1a-2ae4-6397-be824761%22%5D%2C%22qt%22%3A%5B%22UI%22%2C%22EXTERNAL%22%2C%22ACCELERATION%22%5D%7D",
"isArrowCachingEnabled": false,
"isCanView": true,
"isCanAlter": true,
"isEnabled": true
}
],
"nextPageToken": "CiQxNjc0MjhlYi03OTM2LTRlYTItYTFmYi0yM2IxYWM2ZTk0NTQSAA==",
"isCanAlterReflections": true
}
This endpoint supports query parameters that you can add to the request URL to include only specific types of reflections in the reflection summary, specify the maximum number of results to return, and sort the response to list reflections in ascending or descending order.
For example, to order the reflections within the summary in ascending order by reflectionName, add ?orderBy=reflectionName to the request URL. For descending order, add a - character before the attribute name: ?orderBy=-reflectionName.
In the same request, you can add the filter query parameter to retrieve only the raw reflections that are refreshed manually or by schedule, are enabled, and apply to datasets with samples.dremio.com in their paths. The JSON object for such a filter would look like this:
{
"reflectionType": ["RAW"],
"refreshStatus": ["MANUAL","SCHEDULED"],
"enabledFlag": true,
"reflectionNameOrDatasetPath": "samples.dremio.com"
}
However, to use the JSON object in the request URL, you must convert it to URL-encoded JSON, which looks like this:
Example JSON Object in URL-Encoded JSON%7B%0A%20%20%22reflectionType%22%3A%20%5B%22RAW%22%5D%2C%0A%20%20%22refreshStatus%22%3A%20%5B%22MANUAL%22%2C%22SCHEDULED%22%5D%2C%0A%20%20%22enabledFlag%22%3A%20true%2C%0A%20%20%22reflectionNameOrDatasetPath%22%3A%20%22samples.dremio.com%22%0A%7D
Here is an example request URL that includes both the orderBy and filter query parameters:
curl -X GET 'https://api.dremio.cloud/v0/api/projects/{project-id}/reflection-summary?orderBy=reflectionName&filter=%7B%0A%20%20%22reflectionType%22%3A%20%5B%22RAW%22%5D%2C%0A%20%20%22refreshStatus%22%3A%20%5B%22MANUAL%22%2C%22SCHEDULED%22%5D%2C%0A%20%20%22enabledFlag%22%3A%20true%2C%0A%20%20%22reflectionNameOrDatasetPath%22%3A%20%22samples.dremio.com%22%0A%7D' \
--header 'Authorization: Bearer <PersonalAccessToken>' \
--header 'Content-Type: application/json'
For this request, the reflection summary in the response will include only raw reflections that are refreshed manually or by schedule, are enabled, and apply to datasets with samples.dremio.com in their paths, and the reflections will be listed in ascending order by reflectionName.
Response Status Codes
{{< api-responses file="data/api/reflection.json" data="retrieveReflectionSummaryResponseCodes" >}}