Outline ![]()
Files ![]()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
/*
* SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <stdlib.h>
#include <string.h>
#include <sys/cdefs.h>
#include "soc/soc_caps.h"
#include "esp_log.h"
#include "esp_check.h"
#include "esp_memory_utils.h"
#include "esp_heap_caps.h"
#include "esp_private/gdma_link.h"
#include "hal/cache_hal.h"
#include "hal/cache_ll.h"
#include "esp_cache.h"
static const char *TAG = "gdma";
#if SOC_NON_CACHEABLE_OFFSET
#define GDMA_CACHE_ADDR_TO_NON_CACHE_ADDR(addr) ((addr) + SOC_NON_CACHEABLE_OFFSET)
#else
#define GDMA_CACHE_ADDR_TO_NON_CACHE_ADDR(addr) (addr)
#endif
#define ALIGN_UP(num, align) (((num) + ((align) - 1)) & ~((align) - 1))
#define ALIGN_DOWN(num, align) ((num) & ~((align) - 1))
// GDMA link list item definition
// TODO: this type will eventually become target specific, we need to move it to the LL layer or soc layer
typedef struct gdma_link_list_item_t gdma_link_list_item_t;
struct gdma_link_list_item_t {
struct {
uint32_t size : 12; /*!< Buffer size */
uint32_t length : 12; /*!< Number of valid bytes in the buffer */
uint32_t reversed24 : 4; /*!< Reserved */
uint32_t err_eof : 1; /*!< Whether the received buffer contains error, the error was reported by the peripheral */
uint32_t reserved29 : 1; /*!< Reserved */
uint32_t suc_eof : 1; /*!< Whether the list item should notify the peripheral an "EOF" event */
uint32_t owner : 1; /*!< Who is allowed to access the buffer */
} dw0; /*!< list item Word 0 */
void *buffer; /*!< Pointer to the buffer */
gdma_link_list_item_t *next; /*!< Pointer to the next list item (set to NULL if the list item is the last one of the link) */
};
///< Maximum size of the buffer that can be carried by a DMA link list item
#define GDMA_MAX_BUFFER_SIZE_PER_LINK_ITEM 4095
typedef struct gdma_link_list_t {
uint32_t num_items; // number of items in the link list
size_t item_size; // size of each item
size_t buffer_alignment; // Alignment of each buffer
uint8_t *items; // pointer to the link list items
uint8_t *items_nc; // pointer to the link list items, non-cached
struct {
uint32_t check_owner: 1; // Whether the link list is responsible for checking the ownership when mount data buffers
} flags;
} gdma_link_list_t;
esp_err_t gdma_new_link_list(const gdma_link_list_config_t *config, gdma_link_list_handle_t *ret_list)
{
esp_err_t ret = ESP_OK;
uint8_t *items = NULL;
gdma_link_list_t *list = NULL;
ESP_RETURN_ON_FALSE(config && ret_list, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
ESP_RETURN_ON_FALSE(config->num_items, ESP_ERR_INVALID_ARG, TAG, "invalid number of items");
size_t buffer_alignment = config->buffer_alignment;
if (buffer_alignment == 0) {
buffer_alignment = 1;
}
ESP_RETURN_ON_FALSE((buffer_alignment & (buffer_alignment - 1)) == 0, ESP_ERR_INVALID_ARG, TAG, "invalid buffer alignment: %zu", buffer_alignment);
// the link list container is allocated from internal memory
list = heap_caps_calloc(1, sizeof(gdma_link_list_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
ESP_GOTO_ON_FALSE(list, ESP_ERR_NO_MEM, err, TAG, "no mem for link list");
uint32_t num_items = config->num_items;
size_t item_alignment = config->item_alignment ? config->item_alignment : 4;
// each list item should align to the specified alignment
size_t item_size = ALIGN_UP(sizeof(gdma_link_list_item_t), item_alignment);
uint32_t list_items_mem_caps = MALLOC_CAP_8BIT | MALLOC_CAP_DMA;
if (config->flags.items_in_ext_mem) {
list_items_mem_caps |= MALLOC_CAP_SPIRAM;
} else {
list_items_mem_caps |= MALLOC_CAP_INTERNAL;
}
items = heap_caps_aligned_calloc(item_alignment, num_items, item_size, list_items_mem_caps);
ESP_GOTO_ON_FALSE(items, ESP_ERR_NO_MEM, err, TAG, "no mem for link list items");
// do memory sync if the list items are in the cache
uint32_t data_cache_line_size = 0;
if (config->flags.items_in_ext_mem) {
data_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_DATA);
} else {
data_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA);
}
if (data_cache_line_size) {
// write back and then invalidate the cache, because later we will read/write the link list items by non-cached address
ESP_GOTO_ON_ERROR(esp_cache_msync(items, ALIGN_UP(num_items * item_size, data_cache_line_size),
ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_INVALIDATE),
err, TAG, "cache sync failed");
}
list->num_items = num_items;
list->item_size = item_size;
list->items = items;
// calculate the non-cached address
list->items_nc = GDMA_CACHE_ADDR_TO_NON_CACHE_ADDR(items);
list->buffer_alignment = buffer_alignment;
list->flags.check_owner = config->flags.check_owner;
ESP_LOGD(TAG, "new link list @%p, items @%p", list, items);
*ret_list = list;
return ESP_OK;
err:
if (list) {
free(list);
}
if (items) {
free(items);
}
return ret;
}
esp_err_t gdma_del_link_list(gdma_link_list_handle_t list)
{
ESP_RETURN_ON_FALSE(list, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
ESP_LOGD(TAG, "del link list at %p", list);
free(list->items);
free(list);
return ESP_OK;
}
esp_err_t gdma_link_mount_buffers(gdma_link_list_handle_t list, int start_item_index, const gdma_buffer_mount_config_t *buf_config_array, size_t num_buf, int *end_item_index)
{
ESP_RETURN_ON_FALSE(list && buf_config_array && num_buf, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
size_t buffer_alignment = list->buffer_alignment;
size_t item_size = list->item_size;
uint32_t list_item_capacity = list->num_items;
// ensure the start_item_index is between 0 and `list_item_capacity - 1`
start_item_index = (start_item_index % list_item_capacity + list_item_capacity) % list_item_capacity;
size_t max_buffer_mount_length = ALIGN_DOWN(GDMA_MAX_BUFFER_SIZE_PER_LINK_ITEM, buffer_alignment);
uint32_t begin_item_idx = start_item_index;
gdma_link_list_item_t *lli_nc = NULL;
uint32_t num_items_avail = 0;
// if the link list is responsible for checking the ownership, we need to skip the items that are owned by the DMA
if (list->flags.check_owner) {
for (int i = 0; i < list_item_capacity; i++) {
lli_nc = (gdma_link_list_item_t *)(list->items_nc + (i + start_item_index) % list_item_capacity * item_size);
if (lli_nc->dw0.owner == GDMA_LLI_OWNER_CPU) {
num_items_avail++;
}
}
} else {
num_items_avail = list_item_capacity;
}
// check alignment and length for each buffer
for (size_t bi = 0; bi < num_buf; bi++) {
const gdma_buffer_mount_config_t *config = &buf_config_array[bi];
uint8_t *buf = (uint8_t *)config->buffer;
size_t len = config->length;
// check the buffer alignment
ESP_RETURN_ON_FALSE(((uintptr_t)buf & (buffer_alignment - 1)) == 0, ESP_ERR_INVALID_ARG, TAG, "buffer not aligned to %zu", buffer_alignment);
uint32_t num_items_need = (len + max_buffer_mount_length - 1) / max_buffer_mount_length;
// check if there are enough link list items
ESP_RETURN_ON_FALSE((begin_item_idx + num_items_need) <= (start_item_index + num_items_avail), ESP_ERR_INVALID_ARG, TAG, "no more space for buffer mounting");
begin_item_idx += num_items_need;
}
// link_nodes[start_item_index-1] --> link_nodes[start_item_index]
lli_nc = (gdma_link_list_item_t *)(list->items_nc + (start_item_index + list_item_capacity - 1) % list_item_capacity * item_size);
lli_nc->next = (gdma_link_list_item_t *)(list->items + start_item_index * item_size);
begin_item_idx = start_item_index;
for (size_t bi = 0; bi < num_buf; bi++) {
const gdma_buffer_mount_config_t *config = &buf_config_array[bi];
uint8_t *buf = (uint8_t *)config->buffer;
size_t len = config->length;
// skip zero-length buffer
if (len == 0 || buf == NULL) {
continue;
}
uint32_t num_items_need = (len + max_buffer_mount_length - 1) / max_buffer_mount_length;
// mount the buffer to the link list
for (int i = 0; i < num_items_need; i++) {
lli_nc = (gdma_link_list_item_t *)(list->items_nc + (i + begin_item_idx) % list_item_capacity * item_size);
lli_nc->buffer = buf;
lli_nc->dw0.length = len > max_buffer_mount_length ? max_buffer_mount_length : len;
// in fact the DMA doesn't check the "size" field, but we still set it to "length" for consistency
// it's the user's responsibility to make sure the buffer size is sufficient
lli_nc->dw0.size = lli_nc->dw0.length;
// mark the EOF node
lli_nc->dw0.suc_eof = (config->flags.mark_eof == 1) && (i == num_items_need - 1);
// mark the final node
if ((config->flags.mark_final == 1) && (i == num_items_need - 1)) {
lli_nc->next = NULL;
} else {
lli_nc->next = (gdma_link_list_item_t *)(list->items + (i + begin_item_idx + 1) % list_item_capacity * item_size);
}
lli_nc->dw0.owner = GDMA_LLI_OWNER_DMA;
buf += max_buffer_mount_length;
len -= max_buffer_mount_length;
}
begin_item_idx += num_items_need;
}
// return the index of the last modified list item
if (end_item_index) {
*end_item_index = (begin_item_idx - 1 + list_item_capacity) % list_item_capacity;
}
return ESP_OK;
}
uintptr_t gdma_link_get_head_addr(gdma_link_list_handle_t list)
{
ESP_RETURN_ON_FALSE(list, 0, TAG, "invalid argument");
return (uintptr_t)(list->items);
}
esp_err_t gdma_link_concat(gdma_link_list_handle_t first_link, int first_link_item_index, gdma_link_list_handle_t second_link, int second_link_item_index)
{
ESP_RETURN_ON_FALSE(first_link && second_link, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
gdma_link_list_item_t *lli_nc = NULL;
// ensure the first_link_item_index is between 0 and `num_items - 1`
int num_items = first_link->num_items;
first_link_item_index = (first_link_item_index % num_items + num_items) % num_items;
lli_nc = (gdma_link_list_item_t *)(first_link->items_nc + first_link_item_index * first_link->item_size);
// ensure the second_link_item_index is between 0 and `num_items - 1`
num_items = second_link->num_items;
second_link_item_index = (second_link_item_index % num_items + num_items) % num_items;
// concatenate the two link lists
lli_nc->next = (gdma_link_list_item_t *)(second_link->items + second_link_item_index * second_link->item_size);
return ESP_OK;
}
esp_err_t gdma_link_set_owner(gdma_link_list_handle_t list, int item_index, gdma_lli_owner_t owner)
{
ESP_RETURN_ON_FALSE_ISR(list, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
int num_items = list->num_items;
// ensure the item_index is between 0 and `num_items - 1`
item_index = (item_index % num_items + num_items) % num_items;
gdma_link_list_item_t *lli = (gdma_link_list_item_t *)(list->items_nc + item_index * list->item_size);
lli->dw0.owner = owner;
return ESP_OK;
}
esp_err_t gdma_link_get_owner(gdma_link_list_handle_t list, int item_index, gdma_lli_owner_t *owner)
{
ESP_RETURN_ON_FALSE_ISR(list && owner, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
int num_items = list->num_items;
// ensure the item_index is between 0 and `num_items - 1`
item_index = (item_index % num_items + num_items) % num_items;
gdma_link_list_item_t *lli = (gdma_link_list_item_t *)(list->items_nc + item_index * list->item_size);
*owner = lli->dw0.owner;
return ESP_OK;
}
size_t gdma_link_count_buffer_size_till_eof(gdma_link_list_handle_t list, int start_item_index)
{
if (!list) {
return 0;
}
int num_items = list->num_items;
// ensure the start_item_index is between 0 and `num_items - 1`
start_item_index = (start_item_index % num_items + num_items) % num_items;
size_t buf_size = 0;
gdma_link_list_item_t *lli_nc = NULL;
for (int i = 0; i < num_items; i++) {
lli_nc = (gdma_link_list_item_t *)(list->items_nc + (start_item_index + i) % num_items * list->item_size);
buf_size += lli_nc->dw0.length;
// break if the current item is the last one or the EOF item
if (lli_nc->dw0.suc_eof || lli_nc->next == NULL) {
break;
}
}
return buf_size;
}
Details ![]()
Show: from Types: Columns: ![]( "Show Project Names")
![]( "Show Locations")
![]( "Show Referring Symbols")
![]( "Show Scope")
This file uses the notable symbols shown below. Click anywhere in the file to view more details.