Outline ![]()
Files ![]()
loading (1/5)...
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
/*
* SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <sys/param.h>
#include <inttypes.h>
#include <string.h>
#include "sdkconfig.h"
#include "esp_check.h"
#include "esp_log.h"
#include "esp_heap_caps.h"
#include "esp_memory_utils.h"
#include "esp_dma_utils.h"
#include "esp_private/esp_dma_utils.h"
#include "esp_private/esp_cache_private.h"
#include "soc/soc_caps.h"
#include "hal/hal_utils.h"
#include "hal/cache_hal.h"
#include "hal/cache_ll.h"
#include "esp_cache.h"
static const char *TAG = "dma_utils";
#define ALIGN_UP_BY(num, align) (((num) + ((align) - 1)) & ~((align) - 1))
esp_err_t esp_dma_split_rx_buffer_to_cache_aligned(void *rx_buffer, size_t buffer_len, dma_buffer_split_array_t *align_buf_array, uint8_t** ret_stash_buffer)
{
esp_err_t ret = ESP_OK;
uint8_t* stash_buffer = NULL;
ESP_RETURN_ON_FALSE(rx_buffer && buffer_len && align_buf_array, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
// read the cache line size of internal and external memory, we also use this information to check if a given memory is behind the cache
size_t int_mem_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA);
size_t ext_mem_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_DATA);
size_t split_line_size = 0;
if (esp_ptr_external_ram(rx_buffer)) {
split_line_size = ext_mem_cache_line_size;
} else if (esp_ptr_internal(rx_buffer)) {
split_line_size = int_mem_cache_line_size;
}
ESP_LOGV(TAG, "split_line_size:%zu", split_line_size);
// allocate the stash buffer from internal RAM
// Note, the split_line_size can be 0, in this case, the stash_buffer is also NULL, which is fine
stash_buffer = heap_caps_calloc(2, split_line_size, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
ESP_RETURN_ON_FALSE(!(split_line_size && !stash_buffer), ESP_ERR_NO_MEM, TAG, "no mem for stash buffer");
// clear align_array to avoid garbage data
memset(align_buf_array, 0, sizeof(dma_buffer_split_array_t));
bool need_cache_sync[3] = {false};
// if split_line_size is non-zero, split the buffer into head, body and tail
if (split_line_size > 0) {
// calculate head_overflow_len
size_t head_overflow_len = (uintptr_t)rx_buffer % split_line_size;
head_overflow_len = head_overflow_len ? split_line_size - head_overflow_len : 0;
ESP_LOGV(TAG, "head_addr:%p head_overflow_len:%zu", rx_buffer, head_overflow_len);
// calculate tail_overflow_len
size_t tail_overflow_len = ((uintptr_t)rx_buffer + buffer_len) % split_line_size;
ESP_LOGV(TAG, "tail_addr:%p tail_overflow_len:%zu", rx_buffer + buffer_len - tail_overflow_len, tail_overflow_len);
uint8_t extra_buf_count = 0;
uint8_t* input_buffer = (uint8_t*)rx_buffer;
align_buf_array->buf.head.recovery_address = input_buffer;
align_buf_array->buf.head.aligned_buffer = stash_buffer + split_line_size * extra_buf_count++;
align_buf_array->buf.head.length = head_overflow_len;
need_cache_sync[0] = int_mem_cache_line_size > 0;
align_buf_array->buf.body.recovery_address = input_buffer + head_overflow_len;
align_buf_array->buf.body.aligned_buffer = input_buffer + head_overflow_len;
align_buf_array->buf.body.length = buffer_len - head_overflow_len - tail_overflow_len;
need_cache_sync[1] = true;
align_buf_array->buf.tail.recovery_address = input_buffer + buffer_len - tail_overflow_len;
align_buf_array->buf.tail.aligned_buffer = stash_buffer + split_line_size * extra_buf_count++;
align_buf_array->buf.tail.length = tail_overflow_len;
need_cache_sync[2] = int_mem_cache_line_size > 0;
// special handling when input_buffer length is no more than buffer alignment
if (head_overflow_len >= buffer_len || tail_overflow_len >= buffer_len) {
align_buf_array->buf.head.length = buffer_len ;
align_buf_array->buf.body.length = 0 ;
align_buf_array->buf.tail.length = 0 ;
}
} else {
align_buf_array->buf.body.aligned_buffer = rx_buffer;
align_buf_array->buf.body.recovery_address = rx_buffer;
align_buf_array->buf.body.length = buffer_len;
need_cache_sync[1] = false;
}
for (int i = 0; i < 3; i++) {
if (align_buf_array->aligned_buffer[i].length == 0) {
align_buf_array->aligned_buffer[i].aligned_buffer = NULL;
align_buf_array->aligned_buffer[i].recovery_address = NULL;
need_cache_sync[i] = false;
}
}
// invalidate the aligned buffer if necessary
for (int i = 0; i < 3; i++) {
if (need_cache_sync[i]) {
size_t sync_size = align_buf_array->aligned_buffer[i].length;
if (sync_size < split_line_size) {
// If the buffer is smaller than the cache line size, we need to sync the whole buffer
sync_size = split_line_size;
}
esp_err_t res = esp_cache_msync(align_buf_array->aligned_buffer[i].aligned_buffer, sync_size, ESP_CACHE_MSYNC_FLAG_DIR_M2C);
ESP_GOTO_ON_ERROR(res, err, TAG, "failed to do cache sync");
}
}
*ret_stash_buffer = stash_buffer;
return ESP_OK;
err:
if (stash_buffer) {
free(stash_buffer);
}
return ret;
}
esp_err_t esp_dma_merge_aligned_rx_buffers(dma_buffer_split_array_t *align_array)
{
ESP_RETURN_ON_FALSE_ISR(align_array, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
// only need to copy the head and tail buffer
if (align_array->buf.head.length) {
memcpy(align_array->buf.head.recovery_address, align_array->buf.head.aligned_buffer, align_array->buf.head.length);
}
if (align_array->buf.tail.length) {
memcpy(align_array->buf.tail.recovery_address, align_array->buf.tail.aligned_buffer, align_array->buf.tail.length);
}
return ESP_OK;
}
size_t esp_dma_calculate_node_count(size_t buffer_size, size_t buffer_alignment, size_t max_buffer_size_per_node)
{
// buffer_alignment should be power of 2
ESP_RETURN_ON_FALSE(buffer_alignment && ((buffer_alignment & (buffer_alignment - 1)) == 0), 0, TAG, "invalid buffer alignment");
// align down the max_buffer_size_per_node
max_buffer_size_per_node = max_buffer_size_per_node & ~(buffer_alignment - 1);
// calculate the number of nodes
return (buffer_size + max_buffer_size_per_node - 1) / max_buffer_size_per_node;
}
esp_err_t esp_dma_capable_malloc(size_t size, const esp_dma_mem_info_t *dma_mem_info, void **out_ptr, size_t *actual_size)
{
ESP_RETURN_ON_FALSE_ISR(dma_mem_info && out_ptr, ESP_ERR_INVALID_ARG, TAG, "null pointer");
size_t alignment_bytes = 0;
//dma align
size_t dma_alignment_bytes = dma_mem_info->dma_alignment_bytes;
//cache align
int cache_flags = 0;
size_t cache_alignment_bytes = 0;
int heap_caps = dma_mem_info->extra_heap_caps | MALLOC_CAP_DMA;
if (dma_mem_info->extra_heap_caps & MALLOC_CAP_SPIRAM) {
cache_flags |= MALLOC_CAP_SPIRAM;
heap_caps = dma_mem_info->extra_heap_caps | MALLOC_CAP_SPIRAM;
/**
* This is a workaround because we don't have `MALLOC_CAP_DMA | MALLOC_CAP_SPIRAM`
* match when using heap_cap related allocations.
*/
heap_caps &= ~MALLOC_CAP_DMA;
}
// Return value unused if asserts are disabled
esp_err_t __attribute((unused)) ret = esp_cache_get_alignment(cache_flags, &cache_alignment_bytes);
assert(ret == ESP_OK);
//Get the least common multiple of two alignment
alignment_bytes = hal_utils_calc_lcm(dma_alignment_bytes, cache_alignment_bytes);
//malloc
size = ALIGN_UP_BY(size, alignment_bytes);
void *ptr = heap_caps_aligned_alloc(alignment_bytes, size, heap_caps);
ESP_RETURN_ON_FALSE_ISR(ptr, ESP_ERR_NO_MEM, TAG, "Not enough heap memory");
*out_ptr = ptr;
if (actual_size) {
*actual_size = size;
}
return ESP_OK;
}
esp_err_t esp_dma_capable_calloc(size_t calloc_num, size_t size, const esp_dma_mem_info_t *dma_mem_info, void **out_ptr, size_t *actual_size)
{
esp_err_t ret = ESP_FAIL;
size_t size_bytes = 0;
bool ovf = false;
ovf = __builtin_mul_overflow(calloc_num, size, &size_bytes);
ESP_RETURN_ON_FALSE_ISR(!ovf, ESP_ERR_INVALID_ARG, TAG, "wrong size, total size overflow");
void *ptr = NULL;
ret = esp_dma_capable_malloc(size_bytes, dma_mem_info, &ptr, actual_size);
if (ret == ESP_OK) {
memset(ptr, 0, size_bytes);
*out_ptr = ptr;
}
return ret;
}
static bool s_buf_in_region(const void *ptr, size_t size, esp_dma_buf_location_t location)
{
bool found = false;
if (location == ESP_DMA_BUF_LOCATION_INTERNAL) {
if (esp_ptr_dma_capable(ptr) && esp_ptr_dma_capable(ptr + size - 1)) {
found = true;
}
} else if (location == ESP_DMA_BUF_LOCATION_PSRAM) {
#if SOC_PSRAM_DMA_CAPABLE
if (esp_ptr_external_ram(ptr) && esp_ptr_external_ram(ptr + size - 1)) {
found = true;
}
#endif
}
return found;
}
static inline bool s_is_buf_aligned(intptr_t ptr, size_t alignment)
{
return (ptr % alignment == 0);
}
bool esp_dma_is_buffer_alignment_satisfied(const void *ptr, size_t size, esp_dma_mem_info_t dma_mem_info)
{
assert(ptr);
bool found = false;
for (int i = ESP_DMA_BUF_LOCATION_INTERNAL; i < ESP_DMA_BUF_LOCATION_AUTO; i++) {
if (s_buf_in_region(ptr, size, i)) {
found = true;
break;
}
}
if (!found) {
return false;
}
size_t alignment_bytes = 0;
//dma align
size_t dma_alignment_bytes = dma_mem_info.dma_alignment_bytes;
//cache align
int cache_flags = 0;
size_t cache_alignment_bytes = 0;
if (esp_ptr_external_ram(ptr)) {
cache_flags |= MALLOC_CAP_SPIRAM;
}
// Return value unused if asserts are disabled
esp_err_t __attribute__((unused)) ret = esp_cache_get_alignment(cache_flags, &cache_alignment_bytes);
assert(ret == ESP_OK);
//Get the least common multiple of two alignment
alignment_bytes = hal_utils_calc_lcm(dma_alignment_bytes, cache_alignment_bytes);
bool is_aligned = s_is_buf_aligned((intptr_t)ptr, alignment_bytes) && s_is_buf_aligned((intptr_t)size, alignment_bytes);
return is_aligned;
}
//-----------------------Deprecated APIs-----------------------//
esp_err_t s_legacy_malloc(size_t size, uint32_t flags, void **out_ptr, size_t *actual_size)
{
ESP_RETURN_ON_FALSE_ISR(out_ptr, ESP_ERR_INVALID_ARG, TAG, "null pointer");
int heap_caps = 0;
if (flags & ESP_DMA_MALLOC_FLAG_PSRAM) {
heap_caps |= MALLOC_CAP_SPIRAM;
} else {
heap_caps |= MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL;
}
esp_dma_mem_info_t dma_mem_info = {
.extra_heap_caps = heap_caps,
.dma_alignment_bytes = 4, //legacy API behaviour is only check max dma buffer alignment
};
ESP_RETURN_ON_ERROR_ISR(esp_dma_capable_malloc(size, &dma_mem_info, out_ptr, actual_size), TAG, "failed to do malloc");
return ESP_OK;
}
esp_err_t esp_dma_malloc(size_t size, uint32_t flags, void **out_ptr, size_t *actual_size)
{
return s_legacy_malloc(size, flags, out_ptr, actual_size);
}
esp_err_t esp_dma_calloc(size_t n, size_t size, uint32_t flags, void **out_ptr, size_t *actual_size)
{
ESP_RETURN_ON_FALSE_ISR(out_ptr, ESP_ERR_INVALID_ARG, TAG, "null pointer");
esp_err_t ret = ESP_FAIL;
size_t size_bytes = 0;
bool ovf = false;
ovf = __builtin_mul_overflow(n, size, &size_bytes);
ESP_RETURN_ON_FALSE_ISR(!ovf, ESP_ERR_INVALID_ARG, TAG, "wrong size, total size overflow");
void *ptr = NULL;
ret = s_legacy_malloc(size_bytes, flags, &ptr, actual_size);
if (ret == ESP_OK) {
memset(ptr, 0, size_bytes);
*out_ptr = ptr;
}
return ret;
}
static bool s_buf_in_region_legacy(const void *ptr, size_t size, esp_dma_buf_location_t location, int *heap_caps)
{
bool found = false;
if (location == ESP_DMA_BUF_LOCATION_INTERNAL) {
if (esp_ptr_dma_capable(ptr) && esp_ptr_dma_capable(ptr + size - 1)) {
*heap_caps = MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL;
found = true;
}
} else if (location == ESP_DMA_BUF_LOCATION_PSRAM) {
#if SOC_PSRAM_DMA_CAPABLE
if (esp_ptr_external_ram(ptr) && esp_ptr_external_ram(ptr + size - 1)) {
*heap_caps = MALLOC_CAP_SPIRAM;
found = true;
}
#endif
}
return found;
}
bool esp_dma_is_buffer_aligned(const void *ptr, size_t size, esp_dma_buf_location_t location)
{
assert(ptr);
bool found = false;
int heap_caps = 0;
if (location == ESP_DMA_BUF_LOCATION_AUTO) {
for (int i = ESP_DMA_BUF_LOCATION_INTERNAL; i < ESP_DMA_BUF_LOCATION_AUTO; i++) {
if (s_buf_in_region_legacy(ptr, size, i, &heap_caps)) {
found = true;
break;
}
}
} else if (location == ESP_DMA_BUF_LOCATION_INTERNAL) {
found = s_buf_in_region_legacy(ptr, size, ESP_DMA_BUF_LOCATION_INTERNAL, &heap_caps);
} else {
found = s_buf_in_region_legacy(ptr, size, ESP_DMA_BUF_LOCATION_PSRAM, &heap_caps);
}
if (!found) {
return false;
}
esp_dma_mem_info_t dma_mem_info = {
.extra_heap_caps = heap_caps,
.dma_alignment_bytes = 4, //legacy API behaviour is only check max dma buffer alignment
};
return esp_dma_is_buffer_alignment_satisfied(ptr, size, dma_mem_info);
}
Details ![]()
Show: from Types: Columns: ![]( "Show Project Names")
![]( "Show Locations")
![]( "Show Referring Symbols")
![]( "Show Scope")
This file uses the notable symbols shown below. Click anywhere in the file to view more details.